From 522d829b51d55703d604fa6a2177d1ec6ece4586 Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Mon, 20 Sep 2021 09:15:23 +0200 Subject: [PATCH] import ceph 16.2.6 Signed-off-by: Thomas Lamprecht --- ceph/.github/pull_request_template.md | 1 + ceph/CMakeLists.txt | 2 +- ceph/PendingReleaseNotes | 38 +- ceph/README.aix | 1 - ceph/ceph.spec | 47 +- ceph/ceph.spec.in | 41 +- ceph/changelog.upstream | 5 +- ceph/cmake/modules/BuildBoost.cmake | 2 +- ceph/cmake/modules/CephChecks.cmake | 3 +- ceph/cmake/modules/Findfmt.cmake | 22 +- ceph/debian/control | 8 +- ceph/doc/_ext/ceph_commands.py | 1 + ceph/doc/ceph-volume/index.rst | 3 + ceph/doc/ceph-volume/lvm/index.rst | 6 + ceph/doc/ceph-volume/lvm/migrate.rst | 47 + ceph/doc/ceph-volume/lvm/newdb.rst | 11 + ceph/doc/ceph-volume/lvm/newwal.rst | 11 + ceph/doc/cephadm/client-setup.rst | 53 +- ceph/doc/cephadm/host-management.rst | 47 +- ceph/doc/cephadm/install.rst | 5 + ceph/doc/cephadm/mon.rst | 171 +- ceph/doc/cephadm/monitoring.rst | 6 - ceph/doc/cephadm/operations.rst | 424 +- ceph/doc/cephadm/osd.rst | 5 +- ceph/doc/cephadm/rgw.rst | 35 + ceph/doc/cephadm/service-management.rst | 105 +- ceph/doc/cephadm/troubleshooting.rst | 84 +- ceph/doc/cephadm/upgrade.rst | 46 +- ceph/doc/cephfs/administration.rst | 29 +- ceph/doc/cephfs/fs-nfs-exports.rst | 47 + ceph/doc/cephfs/upgrading.rst | 39 +- ceph/doc/dev/cephadm/developing-cephadm.rst | 14 + ceph/doc/dev/developer_guide/dash-devel.rst | 82 +- ceph/doc/install/containers.rst | 11 +- ceph/doc/man/8/ceph-volume.rst | 92 +- ceph/doc/man/8/cephadm.rst | 10 +- ceph/doc/mgr/dashboard.rst | 48 +- ceph/doc/mgr/dashboard_plugins/motd.inc.rst | 30 + ceph/doc/rados/operations/balancer.rst | 33 +- ceph/doc/rados/operations/monitoring.rst | 7 +- .../doc/rados/operations/placement-groups.rst | 40 +- ceph/doc/radosgw/frontends.rst | 33 + ceph/doc/radosgw/vault.rst | 13 + ceph/make-debs.sh | 18 +- ceph/make-dist | 2 +- ceph/monitoring/grafana/build/Makefile | 123 +- .../grafana/dashboards/CMakeLists.txt | 32 +- .../grafana/dashboards/ceph-cluster.json | 72 +- .../grafana/dashboards/host-details.json | 2 +- .../jsonnet/grafana_dashboards.jsonnet | 54 + .../dashboards/osd-device-details.json | 2 +- .../grafana/dashboards/pool-detail.json | 2 +- .../grafana/dashboards/pool-overview.json | 2 +- .../dashboards/radosgw-sync-overview.json | 873 ++-- .../grafana/dashboards/rbd-details.json | 818 ++-- .../dashboards/requirements-grafonnet.txt | 1 + .../grafana/dashboards/test-jsonnet.sh | 30 + ceph/monitoring/grafana/dashboards/tox.ini | 22 + .../centos_8.2_container_tools_3.0.yaml | 14 + .../podman/centos_8.2_kubic_stable.yaml | 18 - .../centos_8.3_container_tools_3.0.yaml | 14 + .../rgw/ignore-pg-availability.yaml | 0 .../standalone/osd/osd-bluefs-volume-ops.sh | 77 +- ceph/qa/standalone/osd/osd-force-create-pg.sh | 3 +- ceph/qa/standalone/osd/osd-reuse-id.sh | 3 +- ceph/qa/standalone/osd/pg-split-merge.sh | 5 +- ceph/qa/standalone/scrub/osd-scrub-repair.sh | 200 +- .../suites/fs/functional/tasks/mds-full.yaml | 2 + ceph/qa/suites/fs/upgrade/nofs/% | 0 ceph/qa/suites/fs/upgrade/nofs/.qa | 1 + ceph/qa/suites/fs/upgrade/nofs/README | 3 + .../fs/upgrade/nofs/bluestore-bitmap.yaml | 1 + .../suites/fs/upgrade/nofs/centos_latest.yaml | 1 + ceph/qa/suites/fs/upgrade/nofs/conf | 1 + .../fs/upgrade/nofs/no-mds-cluster.yaml | 6 + ceph/qa/suites/fs/upgrade/nofs/overrides/% | 0 ceph/qa/suites/fs/upgrade/nofs/overrides/.qa | 1 + .../fs/upgrade/nofs/overrides/pg-warn.yaml | 5 + .../nofs/overrides/whitelist_health.yaml | 1 + .../whitelist_wrongly_marked_down.yaml | 1 + ceph/qa/suites/fs/upgrade/nofs/tasks/% | 0 ceph/qa/suites/fs/upgrade/nofs/tasks/.qa | 1 + .../fs/upgrade/nofs/tasks/0-octopus.yaml | 38 + .../fs/upgrade/nofs/tasks/1-upgrade.yaml | 45 + .../centos_8.2_container_tools_3.0.yaml | 1 + .../0-distro/centos_8.2_kubic_stable.yaml | 1 - .../smoke/distro/centos_8.2_kubic_stable.yaml | 1 - .../centos_8.3_container_tools_3.0.yaml | 1 + .../smoke/distro/rhel_8.3_kubic_stable.yaml | 1 - .../centos_8.2_container_tools_3.0.yaml | 1 + .../0-distro/centos_8.2_kubic_stable.yaml | 1 - .../centos_8.2_container_tools_3.0.yaml | 1 + .../rados/dashboard/tasks/dashboard.yaml | 1 + .../rgw/crypt/ignore-pg-availability.yaml | 2 +- .../rgw/multifs/ignore-pg-availability.yaml | 2 +- .../rgw/multisite/ignore-pg-availability.yaml | 2 +- ceph/qa/suites/rgw/multisite/overrides.yaml | 1 + ceph/qa/suites/rgw/sts/.qa | 1 + .../rgw/sts/ignore-pg-availability.yaml | 2 +- .../rgw/verify/ignore-pg-availability.yaml | 2 +- .../point-to-point-upgrade.yaml | 18 +- .../1-ceph-install/pacific..yaml | 6 +- .../6-final-workload/rbd-python.yaml | 2 +- ceph/qa/tasks/ceph_manager.py | 24 +- ceph/qa/tasks/cephfs/caps_helper.py | 2 +- ceph/qa/tasks/cephfs/cephfs_test_case.py | 6 +- ceph/qa/tasks/cephfs/filesystem.py | 72 +- ceph/qa/tasks/cephfs/fuse_mount.py | 2 +- ceph/qa/tasks/cephfs/kernel_mount.py | 97 +- ceph/qa/tasks/cephfs/mount.py | 63 +- ceph/qa/tasks/cephfs/test_admin.py | 283 +- ceph/qa/tasks/cephfs/test_cap_flush.py | 4 +- ceph/qa/tasks/cephfs/test_cephfs_shell.py | 74 +- ceph/qa/tasks/cephfs/test_client_recovery.py | 15 +- ceph/qa/tasks/cephfs/test_data_scan.py | 34 +- ceph/qa/tasks/cephfs/test_failover.py | 4 +- ceph/qa/tasks/cephfs/test_fragment.py | 2 +- ceph/qa/tasks/cephfs/test_full.py | 34 +- ceph/qa/tasks/cephfs/test_mirroring.py | 116 +- ceph/qa/tasks/cephfs/test_misc.py | 4 +- ceph/qa/tasks/cephfs/test_nfs.py | 6 +- ceph/qa/tasks/cephfs/test_scrub_checks.py | 8 +- ceph/qa/tasks/cephfs/test_sessionmap.py | 18 +- ceph/qa/tasks/cephfs/test_volumes.py | 183 +- ceph/qa/tasks/mds_pre_upgrade.py | 20 +- ceph/qa/tasks/mgr/dashboard/test_motd.py | 37 + ceph/qa/tasks/mgr/dashboard/test_rgw.py | 30 +- ceph/qa/tasks/mgr/dashboard/test_settings.py | 12 +- ceph/qa/tasks/mgr/test_insights.py | 11 - ceph/qa/workunits/cephadm/test_cephadm.sh | 3 + ceph/qa/workunits/mon/pg_autoscaler.sh | 18 +- ceph/qa/workunits/mon/test_mon_config_key.py | 38 +- .../rados/test_envlibrados_for_rocksdb.sh | 2 +- ceph/qa/workunits/rbd/qemu-iotests.sh | 6 +- ceph/qa/workunits/rbd/rbd-nbd.sh | 117 +- ceph/qa/workunits/rgw/s3_utilities.pm | 19 +- ceph/run-make-check.sh | 4 +- ceph/src/.git_version | 4 +- ceph/src/CMakeLists.txt | 15 + ceph/src/SimpleRADOSStriper.cc | 2 +- ceph/src/blk/CMakeLists.txt | 2 +- ceph/src/ceph-volume/ceph_volume/api/lvm.py | 14 +- .../ceph_volume/devices/lvm/activate.py | 5 + .../ceph_volume/devices/lvm/batch.py | 7 +- .../ceph_volume/devices/lvm/deactivate.py | 2 - .../ceph_volume/devices/lvm/main.py | 4 + .../ceph_volume/devices/lvm/migrate.py | 690 ++++ .../ceph_volume/devices/raw/list.py | 125 +- .../ceph-volume/ceph_volume/tests/conftest.py | 2 + .../tests/devices/lvm/test_activate.py | 12 + .../tests/devices/lvm/test_batch.py | 9 + .../tests/devices/lvm/test_migrate.py | 2295 +++++++++++ .../tests/devices/raw/test_list.py | 235 ++ .../tests/functional/batch/tox.ini | 12 +- .../lvm/centos8/bluestore/dmcrypt/test.yml | 19 + .../lvm/centos8/filestore/dmcrypt/test.yml | 14 +- .../lvm/playbooks/test_bluestore.yml | 13 + .../lvm/playbooks/test_filestore.yml | 14 +- .../ceph_volume/tests/functional/lvm/tox.ini | 12 +- .../tests/functional/playbooks/deploy.yml | 8 +- .../tests/functional/simple/tox.ini | 10 +- .../tests/util/test_arg_validators.py | 4 +- .../ceph_volume/tests/util/test_device.py | 37 +- .../ceph-volume/ceph_volume/util/device.py | 42 +- ceph/src/ceph-volume/ceph_volume/util/disk.py | 17 +- .../ceph-volume/ceph_volume/util/system.py | 3 +- ceph/src/cephadm/cephadm | 779 ++-- ceph/src/cephadm/tests/fixtures.py | 20 +- ceph/src/cephadm/tests/test_cephadm.py | 745 +++- ceph/src/cephadm/tests/test_networks.py | 201 + ceph/src/client/Client.cc | 43 +- ceph/src/client/Client.h | 3 +- ceph/src/cls/cmpomap/client.h | 14 +- ceph/src/cls/cmpomap/server.cc | 23 +- ceph/src/cls/rgw/cls_rgw.cc | 50 +- ceph/src/cls/rgw/cls_rgw_types.h | 26 +- ceph/src/common/CMakeLists.txt | 3 +- ceph/src/common/Formatter.cc | 1 + ceph/src/common/buffer.cc | 8 +- ceph/src/common/ipaddr.cc | 118 +- ceph/src/common/legacy_config_opts.h | 5 +- ceph/src/common/options.cc | 33 +- ceph/src/common/pick_address.cc | 311 +- ceph/src/common/pick_address.h | 14 + ceph/src/common/subsys.h | 2 +- ceph/src/compressor/snappy/SnappyCompressor.h | 2 +- ceph/src/include/CompatSet.h | 22 +- ceph/src/include/cephfs/metrics/Types.h | 115 +- ceph/src/include/config-h.in.cmake | 3 + ceph/src/include/denc.h | 1 + ceph/src/include/ipaddr.h | 15 +- ceph/src/krbd.cc | 16 +- ceph/src/kv/RocksDBStore.cc | 260 +- ceph/src/kv/RocksDBStore.h | 8 +- ceph/src/mds/Beacon.cc | 7 +- ceph/src/mds/CDir.cc | 7 + ceph/src/mds/FSMap.cc | 109 +- ceph/src/mds/FSMap.h | 38 +- ceph/src/mds/MDBalancer.cc | 10 +- ceph/src/mds/MDBalancer.h | 2 + ceph/src/mds/MDCache.cc | 6 +- ceph/src/mds/MDSMap.cc | 25 +- ceph/src/mds/MDSMap.h | 15 +- ceph/src/mds/MetricsHandler.cc | 18 +- ceph/src/mds/Server.cc | 5 +- ceph/src/mgr/DaemonServer.cc | 12 +- ceph/src/mon/CMakeLists.txt | 5 +- ceph/src/mon/FSCommands.cc | 166 +- ceph/src/mon/FSCommands.h | 12 +- ceph/src/mon/KVMonitor.cc | 3 + ceph/src/mon/MDSMonitor.cc | 145 +- ceph/src/mon/MDSMonitor.h | 7 + ceph/src/mon/MonCommands.h | 26 +- ceph/src/mon/Monitor.cc | 2 +- ceph/src/mon/OSDMonitor.cc | 71 +- ceph/src/mon/OSDMonitor.h | 11 +- ceph/src/mon/PGMap.cc | 19 +- ceph/src/mon/PaxosService.cc | 19 +- ceph/src/msg/CMakeLists.txt | 1 + ceph/src/msg/async/AsyncMessenger.h | 4 +- ceph/src/mypy.ini | 13 + ceph/src/neorados/CMakeLists.txt | 2 + ceph/src/os/bluestore/BlueFS.cc | 107 +- ceph/src/os/bluestore/BlueFS.h | 6 + ceph/src/os/bluestore/BlueRocksEnv.cc | 2 +- ceph/src/os/bluestore/BlueStore.cc | 258 +- ceph/src/os/bluestore/BlueStore.h | 36 +- ceph/src/os/bluestore/bluestore_tool.cc | 38 +- ceph/src/os/bluestore/bluestore_types.h | 14 +- ceph/src/osd/CMakeLists.txt | 2 +- ceph/src/osd/PeeringState.cc | 12 +- ceph/src/osd/PrimaryLogPG.cc | 2 +- ceph/src/osd/PrimaryLogScrub.cc | 72 +- ceph/src/osd/PrimaryLogScrub.h | 2 - ceph/src/osd/osd_types.h | 4 +- ceph/src/osd/pg_scrubber.cc | 73 +- ceph/src/osd/pg_scrubber.h | 29 +- ceph/src/osdc/Objecter.cc | 7 + ceph/src/osdc/Objecter.h | 6 +- ceph/src/pybind/mgr/CMakeLists.txt | 2 +- ceph/src/pybind/mgr/cephadm/migrations.py | 1 + ceph/src/pybind/mgr/cephadm/module.py | 142 +- ceph/src/pybind/mgr/cephadm/schedule.py | 22 +- ceph/src/pybind/mgr/cephadm/serve.py | 261 +- .../mgr/cephadm/services/cephadmservice.py | 11 +- .../pybind/mgr/cephadm/services/ingress.py | 17 +- ceph/src/pybind/mgr/cephadm/services/iscsi.py | 15 +- .../pybind/mgr/cephadm/services/monitoring.py | 36 +- ceph/src/pybind/mgr/cephadm/services/nfs.py | 2 +- .../services/ingress/keepalived.conf.j2 | 2 +- .../templates/services/nfs/ganesha.conf.j2 | 2 +- .../pybind/mgr/cephadm/tests/test_cephadm.py | 27 +- .../mgr/cephadm/tests/test_scheduling.py | 104 + ceph/src/pybind/mgr/cephadm/upgrade.py | 2 +- .../dashboard/ci/cephadm/bootstrap-cluster.sh | 24 +- .../mgr/dashboard/ci/cephadm/ceph_cluster.yml | 11 +- .../ci/cephadm/run-cephadm-e2e-tests.sh | 76 +- .../mgr/dashboard/ci/cephadm/start-cluster.sh | 86 + ...na_uids.py => check_grafana_dashboards.py} | 4 + .../pybind/mgr/dashboard/controllers/home.py | 2 +- .../dashboard/controllers/perf_counters.py | 66 +- .../pybind/mgr/dashboard/controllers/rgw.py | 3 +- .../cluster/mgr-modules.e2e-spec.ts | 5 - .../01-hosts-force-maintenance.e2e-spec.ts | 12 +- .../orchestrator/01-hosts.e2e-spec.ts | 2 +- .../02-hosts-inventory.e2e-spec.ts | 2 +- .../orchestrator/03-inventory.e2e-spec.ts | 2 +- .../cypress/integration/ui/navigation.po.ts | 2 +- .../dist/en-US/1.1c3aa698fdc6e2e06036.js | 1 - .../dist/en-US/1.6da7b376fa1a8a3df154.js | 1 + ...6c3e77fd4.js => 5.0a363eda73eafe0c0332.js} | 0 .../dist/en-US/6.115992dc55f8e1abedbc.js | 1 + .../dist/en-US/6.93b626d621206efed808.js | 1 - .../dashboard/frontend/dist/en-US/index.html | 2 +- .../dist/en-US/main.686c64fd8301e15fd70a.js | 3 - .../dist/en-US/main.b78c1bf5c30e15315e18.js | 3 + .../en-US/runtime.95ade215c318c70e8872.js | 1 - .../en-US/runtime.fcd694c3eff5ef104b53.js | 1 + .../mgr/dashboard/frontend/i18n.config.json | 2 +- .../frontend/src/app/app-routing.module.ts | 2 +- .../iscsi-target-list.component.html | 3 + .../iscsi-target-list.component.spec.ts | 7 + .../iscsi-target-list.component.ts | 48 +- .../cephfs-directories.component.spec.ts | 34 + .../cephfs-directories.component.ts | 13 +- .../host-details/host-details.component.html | 2 +- .../host-details.component.spec.ts | 2 +- .../cluster/hosts/hosts.component.spec.ts | 6 +- .../app/ceph/cluster/hosts/hosts.component.ts | 6 +- .../service-form/service-form.component.html | 4 +- .../service-form.component.spec.ts | 68 +- .../service-form/service-form.component.ts | 27 +- .../src/app/ceph/rgw/models/rgw-daemon.ts | 1 + .../rgw-bucket-list.component.spec.ts | 2 +- .../rgw-bucket-list.component.ts | 25 +- .../rgw-daemon-details.component.html | 2 +- .../rgw-daemon-details.component.spec.ts | 12 + .../rgw-daemon-details.component.ts | 6 +- .../rgw-user-list.component.spec.ts | 2 +- .../rgw-user-list/rgw-user-list.component.ts | 25 +- .../src/app/core/context/context.component.ts | 2 +- .../navigation/navigation.component.html | 3 +- .../navigation/navigation.component.scss | 4 - .../navigation/navigation.component.spec.ts | 3 +- .../navigation/navigation.component.ts | 10 +- .../src/app/shared/api/motd.service.spec.ts | 34 + .../src/app/shared/api/motd.service.ts | 25 + .../shared/classes/list-with-details.class.ts | 22 + .../alert-panel/alert-panel.component.html | 3 +- .../alert-panel/alert-panel.component.ts | 22 +- .../shared/components/components.module.ts | 7 +- .../components/motd/motd.component.html | 8 + .../components/motd/motd.component.scss | 0 .../components/motd/motd.component.spec.ts | 26 + .../shared/components/motd/motd.component.ts | 33 + .../src/app/shared/forms/cd-validators.ts | 9 + .../src/app/shared/pipes/pipes.module.ts | 10 +- .../shared/pipes/sanitize-html.pipe.spec.ts | 26 + .../app/shared/pipes/sanitize-html.pipe.ts | 13 + .../motd-notification.service.spec.ts | 117 + .../services/motd-notification.service.ts | 82 + .../services/prometheus-alert.service.spec.ts | 14 +- .../services/prometheus-alert.service.ts | 5 +- .../frontend/src/locale/messages.cs.xlf | 2650 ++++++------ .../frontend/src/locale/messages.de-DE.xlf | 3150 +++++++------- .../frontend/src/locale/messages.es-ES.xlf | 3291 ++++++++------- .../frontend/src/locale/messages.fr-FR.xlf | 3284 ++++++++------- .../frontend/src/locale/messages.id-ID.xlf | 2761 +++++++------ .../frontend/src/locale/messages.it-IT.xlf | 3304 ++++++++------- .../frontend/src/locale/messages.ja-JP.xlf | 3359 ++++++++------- .../frontend/src/locale/messages.ko-KR.xlf | 2802 +++++++------ .../frontend/src/locale/messages.pl-PL.xlf | 2777 +++++++------ .../frontend/src/locale/messages.pt-BR.xlf | 3270 ++++++++------- .../frontend/src/locale/messages.zh-CN.xlf | 3642 +++++++++-------- .../frontend/src/locale/messages.zh-TW.xlf | 3403 +++++++-------- ceph/src/pybind/mgr/dashboard/module.py | 27 +- ceph/src/pybind/mgr/dashboard/plugins/motd.py | 98 + ceph/src/pybind/mgr/dashboard/rest_client.py | 16 +- .../mgr/dashboard/run-backend-api-tests.sh | 4 +- .../mgr/dashboard/run-frontend-e2e-tests.sh | 6 +- .../mgr/dashboard/services/access_control.py | 9 +- .../mgr/dashboard/services/ceph_service.py | 27 + .../mgr/dashboard/services/iscsi_cli.py | 2 +- .../mgr/dashboard/services/rgw_client.py | 129 +- ceph/src/pybind/mgr/dashboard/settings.py | 6 +- .../pybind/mgr/dashboard/tests/__init__.py | 4 +- .../dashboard/tests/test_access_control.py | 2 +- .../pybind/mgr/dashboard/tests/test_home.py | 7 + .../pybind/mgr/dashboard/tests/test_iscsi.py | 2 +- .../pybind/mgr/dashboard/tests/test_rgw.py | 4 +- .../mgr/dashboard/tests/test_rgw_client.py | 165 +- .../mgr/dashboard/tests/test_settings.py | 2 +- ceph/src/pybind/mgr/dashboard/tools.py | 37 +- ceph/src/pybind/mgr/dashboard/tox.ini | 5 +- ceph/src/pybind/mgr/insights/module.py | 20 + ceph/src/pybind/mgr/mgr_module.py | 65 +- ceph/src/pybind/mgr/mgr_util.py | 142 +- .../mgr/mirroring/fs/snapshot_mirror.py | 57 +- ceph/src/pybind/mgr/nfs/cluster.py | 2 - ceph/src/pybind/mgr/nfs/export.py | 2 - ceph/src/pybind/mgr/nfs/module.py | 5 +- ceph/src/pybind/mgr/orchestrator/__init__.py | 2 +- .../src/pybind/mgr/orchestrator/_interface.py | 22 +- ceph/src/pybind/mgr/orchestrator/module.py | 11 +- ceph/src/pybind/mgr/pg_autoscaler/__init__.py | 5 + ceph/src/pybind/mgr/pg_autoscaler/module.py | 431 +- .../mgr/pg_autoscaler/tests/__init__.py | 0 .../tests/test_cal_final_pg_target.py | 612 +++ .../{test_autoscaler.py => test_cal_ratio.py} | 3 + .../tests/test_overlapping_roots.py | 479 +++ ceph/src/pybind/mgr/prometheus/module.py | 18 +- ceph/src/pybind/mgr/restful/module.py | 7 +- ceph/src/pybind/mgr/rook/module.py | 62 +- ceph/src/pybind/mgr/rook/rook_cluster.py | 120 +- ceph/src/pybind/mgr/stats/fs/perf_stats.py | 7 +- ceph/src/pybind/mgr/telemetry/module.py | 4 +- .../pybind/mgr/test_orchestrator/module.py | 2 +- ceph/src/pybind/mgr/tests/__init__.py | 6 +- ceph/src/pybind/mgr/tox.ini | 9 + .../src/pybind/mgr/volumes/fs/async_cloner.py | 101 +- ceph/src/pybind/mgr/volumes/fs/async_job.py | 4 +- ceph/src/pybind/mgr/volumes/fs/purge_queue.py | 28 +- ceph/src/pybind/mgr/volumes/fs/volume.py | 6 +- ceph/src/pybind/mgr/volumes/module.py | 11 +- ceph/src/pybind/rbd/rbd.pyx | 8 +- .../ceph/deployment/service_spec.py | 4 +- ceph/src/python-common/ceph/utils.py | 39 + ceph/src/python-common/tox.ini | 2 +- ceph/src/rgw/CMakeLists.txt | 2 +- ceph/src/rgw/librgw.cc | 23 + ceph/src/rgw/rgw-orphan-list | 19 +- ceph/src/rgw/rgw_admin.cc | 58 +- ceph/src/rgw/rgw_asio_frontend.cc | 52 + ceph/src/rgw/rgw_auth.cc | 20 +- ceph/src/rgw/rgw_bucket.cc | 2 +- ceph/src/rgw/rgw_cache.h | 5 + ceph/src/rgw/rgw_common.cc | 159 +- ceph/src/rgw/rgw_common.h | 10 +- ceph/src/rgw/rgw_crypt.cc | 4 +- ceph/src/rgw/rgw_datalog.cc | 15 +- ceph/src/rgw/rgw_datalog.h | 6 +- ceph/src/rgw/rgw_file.cc | 8 +- ceph/src/rgw/rgw_http_client.cc | 17 +- ceph/src/rgw/rgw_http_client.h | 18 + ceph/src/rgw/rgw_iam_policy.cc | 38 +- ceph/src/rgw/rgw_iam_policy.h | 14 +- ceph/src/rgw/rgw_kms.cc | 13 + ceph/src/rgw/rgw_log_backing.h | 12 +- ceph/src/rgw/rgw_main.cc | 2 + ceph/src/rgw/rgw_notify.cc | 68 +- ceph/src/rgw/rgw_notify.h | 7 +- ceph/src/rgw/rgw_obj_manifest.cc | 6 + ceph/src/rgw/rgw_op.cc | 503 ++- ceph/src/rgw/rgw_pubsub.cc | 13 +- ceph/src/rgw/rgw_pubsub_push.cc | 1 + ceph/src/rgw/rgw_quota.cc | 53 +- ceph/src/rgw/rgw_quota.h | 12 +- ceph/src/rgw/rgw_rados.cc | 65 +- ceph/src/rgw/rgw_rados.h | 8 +- ceph/src/rgw/rgw_reshard.cc | 15 +- ceph/src/rgw/rgw_rest_bucket.cc | 4 +- ceph/src/rgw/rgw_rest_client.cc | 2 +- ceph/src/rgw/rgw_rest_pubsub.cc | 2 +- ceph/src/rgw/rgw_rest_role.cc | 22 +- ceph/src/rgw/rgw_rest_s3.cc | 3 + ceph/src/rgw/rgw_sal_rados.cc | 4 +- ceph/src/rgw/rgw_sal_rados.h | 6 +- ceph/src/rgw/rgw_sync.cc | 33 +- ceph/src/rgw/rgw_user.cc | 117 +- ceph/src/rgw/rgw_website.cc | 4 +- ceph/src/rgw/services/svc_notify.cc | 131 +- ceph/src/rgw/services/svc_notify.h | 11 +- ceph/src/rgw/services/svc_sys_obj_cache.cc | 4 +- ceph/src/script/set_up_stretch_mode.sh | 24 + ceph/src/test/bufferlist.cc | 17 + ceph/src/test/cli-integration/rbd/unmap.t | 17 + ceph/src/test/cls_cmpomap/test_cls_cmpomap.cc | 54 +- ceph/src/test/docker-test-helper.sh | 8 +- ceph/src/test/librados/service.cc | 2 +- .../librados_test_stub/LibradosTestStub.cc | 2 +- ceph/src/test/mon/PGMap.cc | 6 +- ceph/src/test/objectstore/store_test.cc | 166 +- ceph/src/test/objectstore/test_bluefs.cc | 66 + ceph/src/test/osd/TestOSDMap.cc | 2 +- .../snapshot/test_mock_Replayer.cc | 54 + ceph/src/test/rbd_mirror/test_main.cc | 3 +- ceph/src/test/rgw/CMakeLists.txt | 2 - ceph/src/test/rgw/amqp_mock.cc | 9 + ceph/src/test/rgw/rgw_multi/tests_ps.py | 44 +- .../test/rgw/test_rgw_dmclock_scheduler.cc | 22 +- ceph/src/test/rgw/test_rgw_iam_policy.cc | 9 +- ceph/src/test/test_ipaddr.cc | 106 +- ceph/src/tools/CMakeLists.txt | 2 +- ceph/src/tools/ceph_monstore_tool.cc | 23 +- ceph/src/tools/cephfs_mirror/Mirror.cc | 20 +- ceph/src/tools/cephfs_mirror/PeerReplayer.cc | 2 +- ceph/src/tools/cephfs_mirror/PeerReplayer.h | 12 +- ceph/src/tools/rbd/action/Ggate.cc | 11 + ceph/src/tools/rbd/action/Nbd.cc | 11 + ceph/src/tools/rbd_mirror/ImageReplayer.cc | 2 - ceph/src/tools/rbd_mirror/InstanceReplayer.cc | 1 + ceph/src/tools/rbd_mirror/Types.h | 9 +- .../image_replayer/journal/Replayer.cc | 12 +- .../image_replayer/snapshot/Replayer.cc | 89 +- .../image_replayer/snapshot/Replayer.h | 11 + ceph/src/tools/rbd_mirror/main.cc | 47 +- ceph/win32_deps_build.sh | 19 +- 467 files changed, 35492 insertions(+), 22824 deletions(-) create mode 100644 ceph/doc/ceph-volume/lvm/migrate.rst create mode 100644 ceph/doc/ceph-volume/lvm/newdb.rst create mode 100644 ceph/doc/ceph-volume/lvm/newwal.rst create mode 100644 ceph/doc/mgr/dashboard_plugins/motd.inc.rst create mode 100644 ceph/monitoring/grafana/dashboards/jsonnet/grafana_dashboards.jsonnet create mode 100644 ceph/monitoring/grafana/dashboards/requirements-grafonnet.txt create mode 100644 ceph/monitoring/grafana/dashboards/test-jsonnet.sh create mode 100644 ceph/monitoring/grafana/dashboards/tox.ini create mode 100644 ceph/qa/distros/podman/centos_8.2_container_tools_3.0.yaml delete mode 100644 ceph/qa/distros/podman/centos_8.2_kubic_stable.yaml create mode 100644 ceph/qa/distros/podman/centos_8.3_container_tools_3.0.yaml rename ceph/qa/{suites => }/rgw/ignore-pg-availability.yaml (100%) create mode 100644 ceph/qa/suites/fs/upgrade/nofs/% create mode 120000 ceph/qa/suites/fs/upgrade/nofs/.qa create mode 100644 ceph/qa/suites/fs/upgrade/nofs/README create mode 120000 ceph/qa/suites/fs/upgrade/nofs/bluestore-bitmap.yaml create mode 120000 ceph/qa/suites/fs/upgrade/nofs/centos_latest.yaml create mode 120000 ceph/qa/suites/fs/upgrade/nofs/conf create mode 100644 ceph/qa/suites/fs/upgrade/nofs/no-mds-cluster.yaml create mode 100644 ceph/qa/suites/fs/upgrade/nofs/overrides/% create mode 120000 ceph/qa/suites/fs/upgrade/nofs/overrides/.qa create mode 100644 ceph/qa/suites/fs/upgrade/nofs/overrides/pg-warn.yaml create mode 120000 ceph/qa/suites/fs/upgrade/nofs/overrides/whitelist_health.yaml create mode 120000 ceph/qa/suites/fs/upgrade/nofs/overrides/whitelist_wrongly_marked_down.yaml create mode 100644 ceph/qa/suites/fs/upgrade/nofs/tasks/% create mode 120000 ceph/qa/suites/fs/upgrade/nofs/tasks/.qa create mode 100644 ceph/qa/suites/fs/upgrade/nofs/tasks/0-octopus.yaml create mode 100644 ceph/qa/suites/fs/upgrade/nofs/tasks/1-upgrade.yaml create mode 120000 ceph/qa/suites/orch/cephadm/dashboard/0-distro/centos_8.2_container_tools_3.0.yaml delete mode 120000 ceph/qa/suites/orch/cephadm/dashboard/0-distro/centos_8.2_kubic_stable.yaml delete mode 120000 ceph/qa/suites/orch/cephadm/smoke/distro/centos_8.2_kubic_stable.yaml create mode 120000 ceph/qa/suites/orch/cephadm/smoke/distro/centos_8.3_container_tools_3.0.yaml delete mode 120000 ceph/qa/suites/orch/cephadm/smoke/distro/rhel_8.3_kubic_stable.yaml create mode 120000 ceph/qa/suites/orch/cephadm/workunits/0-distro/centos_8.2_container_tools_3.0.yaml delete mode 120000 ceph/qa/suites/orch/cephadm/workunits/0-distro/centos_8.2_kubic_stable.yaml create mode 120000 ceph/qa/suites/rados/dashboard/centos_8.2_container_tools_3.0.yaml create mode 120000 ceph/qa/suites/rgw/sts/.qa create mode 100644 ceph/qa/tasks/mgr/dashboard/test_motd.py create mode 100644 ceph/src/ceph-volume/ceph_volume/devices/lvm/migrate.py create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_migrate.py create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/devices/raw/test_list.py create mode 100644 ceph/src/cephadm/tests/test_networks.py create mode 100755 ceph/src/pybind/mgr/dashboard/ci/cephadm/start-cluster.sh rename ceph/src/pybind/mgr/dashboard/ci/{check_grafana_uids.py => check_grafana_dashboards.py} (95%) delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/1.1c3aa698fdc6e2e06036.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/1.6da7b376fa1a8a3df154.js rename ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/{5.7ef5282604f6c3e77fd4.js => 5.0a363eda73eafe0c0332.js} (100%) create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/6.115992dc55f8e1abedbc.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/6.93b626d621206efed808.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/main.686c64fd8301e15fd70a.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/main.b78c1bf5c30e15315e18.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/runtime.95ade215c318c70e8872.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/runtime.fcd694c3eff5ef104b53.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/api/motd.service.spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/api/motd.service.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/components/motd/motd.component.html create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/components/motd/motd.component.scss create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/components/motd/motd.component.spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/components/motd/motd.component.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/sanitize-html.pipe.spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/sanitize-html.pipe.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/services/motd-notification.service.spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/services/motd-notification.service.ts create mode 100644 ceph/src/pybind/mgr/dashboard/plugins/motd.py create mode 100644 ceph/src/pybind/mgr/pg_autoscaler/tests/__init__.py create mode 100644 ceph/src/pybind/mgr/pg_autoscaler/tests/test_cal_final_pg_target.py rename ceph/src/pybind/mgr/pg_autoscaler/tests/{test_autoscaler.py => test_cal_ratio.py} (99%) create mode 100644 ceph/src/pybind/mgr/pg_autoscaler/tests/test_overlapping_roots.py diff --git a/ceph/.github/pull_request_template.md b/ceph/.github/pull_request_template.md index 4d8d4d41a..46ad1113d 100644 --- a/ceph/.github/pull_request_template.md +++ b/ceph/.github/pull_request_template.md @@ -48,6 +48,7 @@ https://raw.githubusercontent.com/ceph/ceph/master/SubmittingPatches.rst - `jenkins test make check arm64` - `jenkins test submodules` - `jenkins test dashboard` +- `jenkins test dashboard cephadm` - `jenkins test api` - `jenkins test docs` - `jenkins render docs` diff --git a/ceph/CMakeLists.txt b/ceph/CMakeLists.txt index 86b32019e..f7056c209 100644 --- a/ceph/CMakeLists.txt +++ b/ceph/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.10.2) # remove cmake/modules/FindPython* once 3.12 is required project(ceph - VERSION 16.0.0 + VERSION 16.2.6 LANGUAGES CXX C ASM) foreach(policy diff --git a/ceph/PendingReleaseNotes b/ceph/PendingReleaseNotes index 88c0137c0..f5ac346a4 100644 --- a/ceph/PendingReleaseNotes +++ b/ceph/PendingReleaseNotes @@ -1,5 +1,13 @@ >=17.0.0 +* `ceph-mgr-modules-core` debian package does not recommend `ceph-mgr-rook` + anymore. As the latter depends on `python3-numpy` which cannot be imported in + different Python sub-interpreters multi-times if the version of + `python3-numpy` is older than 1.19. Since `apt-get` installs the `Recommends` + packages by default, `ceph-mgr-rook` was always installed along with + `ceph-mgr` debian package as an indirect dependency. If your workflow depends + on this behavior, you might want to install `ceph-mgr-rook` separately. + * A new library is available, libcephsqlite. It provides a SQLite Virtual File System (VFS) on top of RADOS. The database and journals are striped over RADOS across multiple objects for virtually unlimited scaling and throughput @@ -9,6 +17,28 @@ that were storing state in RADOS omap, especially without striping which limits scalability. +* MDS upgrades no longer require stopping all standby MDS daemons before + upgrading the sole active MDS for a file system. + +* RGW: It is possible to specify ssl options and ciphers for beast frontend now. + The default ssl options setting is "no_sslv2:no_sslv3:no_tlsv1:no_tlsv1_1". + If you want to return back the old behavior add 'ssl_options=' (empty) to + ``rgw frontends`` configuration. + +* fs: A file system can be created with a specific ID ("fscid"). This is useful + in certain recovery scenarios, e.g., monitor database lost and rebuilt, and + the restored file system is expected to have the same ID as before. + +>=16.2.6 +-------- + +* MGR: The pg_autoscaler has a new default 'scale-down' profile which provides more + performance from the start for new pools (for newly created clusters). + Existing clusters will retain the old behavior, now called the 'scale-up' profile. + For more details, see: + + https://docs.ceph.com/en/latest/rados/operations/placement-groups/ + >=16.0.0 -------- @@ -42,12 +72,6 @@ deprecated and will be removed in a future release. Please use ``nfs cluster rm`` and ``nfs export rm`` instead. -* mgr-pg_autoscaler: Autoscaler will now start out by scaling each - pool to have a full complements of pgs from the start and will only - decrease it when other pools need more pgs due to increased usage. - This improves out of the box performance of Ceph by allowing more PGs - to be created for a given pool. - * CephFS: Disabling allow_standby_replay on a file system will also stop all standby-replay daemons for that file system. @@ -159,6 +183,8 @@ CentOS 7.6 and later. To enable older clients, set ``cephx_require_version`` and ``cephx_service_require_version`` config options to 1. +* rgw: The Civetweb frontend is now deprecated and will be removed in Quincy. + >=15.0.0 -------- diff --git a/ceph/README.aix b/ceph/README.aix index 611941339..2bcec224c 100644 --- a/ceph/README.aix +++ b/ceph/README.aix @@ -19,7 +19,6 @@ The following AIX packages are required for developing and compilation, they hav gettext less perl - gdbm pcre rsync zlib diff --git a/ceph/ceph.spec b/ceph/ceph.spec index 2a07bdb6e..e456f9b13 100644 --- a/ceph/ceph.spec +++ b/ceph/ceph.spec @@ -49,6 +49,8 @@ %bcond_without lttng %bcond_without libradosstriper %bcond_without ocf +%global luarocks_package_name luarocks +%bcond_without lua_packages %global _remote_tarball_prefix https://download.ceph.com/tarballs/ %endif %if 0%{?suse_version} @@ -73,6 +75,21 @@ %if ! %{defined _fillupdir} %global _fillupdir /var/adm/fillup-templates %endif +#luarocks +%if 0%{?is_opensuse} +# openSUSE +%bcond_without lua_packages +%if 0%{?sle_version} +# openSUSE Leap +%global luarocks_package_name lua53-luarocks +%else +# openSUSE Tumbleweed +%global luarocks_package_name lua54-luarocks +%endif +%else +# SLE +%bcond_with lua_packages +%endif %endif %bcond_with seastar %bcond_with jaeger @@ -96,19 +113,6 @@ %endif %endif -%if 0%{?suse_version} -%if !0%{?is_opensuse} -# SLE does not support luarocks -%bcond_with lua_packages -%else -%global luarocks_package_name lua53-luarocks -%bcond_without lua_packages -%endif -%else -%global luarocks_package_name luarocks -%bcond_without lua_packages -%endif - %{!?_udevrulesdir: %global _udevrulesdir /lib/udev/rules.d} %{!?tmpfiles_create: %global tmpfiles_create systemd-tmpfiles --create} %{!?python3_pkgversion: %global python3_pkgversion 3} @@ -122,7 +126,7 @@ # main package definition ################################################################################# Name: ceph -Version: 16.2.5 +Version: 16.2.6 Release: 0%{?dist} %if 0%{?fedora} || 0%{?rhel} Epoch: 2 @@ -138,7 +142,7 @@ License: LGPL-2.1 and LGPL-3.0 and CC-BY-SA-3.0 and GPL-2.0 and BSL-1.0 and BSD- Group: System/Filesystems %endif URL: http://ceph.com/ -Source0: %{?_remote_tarball_prefix}ceph-16.2.5.tar.bz2 +Source0: %{?_remote_tarball_prefix}ceph-16.2.6.tar.bz2 %if 0%{?suse_version} # _insert_obs_source_lines_here ExclusiveArch: x86_64 aarch64 ppc64le s390x @@ -168,7 +172,6 @@ BuildRequires: gcc-toolset-9-gcc-c++ >= 9.2.1-2.3 %else BuildRequires: gcc-c++ %endif -BuildRequires: gdbm %if 0%{with tcmalloc} # libprofiler did not build on ppc64le until 2.7.90 %if 0%{?fedora} || 0%{?rhel} >= 8 @@ -292,7 +295,6 @@ BuildRequires: libbz2-devel BuildRequires: mozilla-nss-devel BuildRequires: keyutils-devel BuildRequires: libopenssl-devel -BuildRequires: lsb-release BuildRequires: openldap2-devel #BuildRequires: krb5 #BuildRequires: krb5-devel @@ -317,7 +319,6 @@ BuildRequires: openldap-devel #BuildRequires: krb5-devel BuildRequires: openssl-devel BuildRequires: CUnit-devel -BuildRequires: redhat-lsb-core BuildRequires: python%{python3_pkgversion}-devel BuildRequires: python%{python3_pkgversion}-setuptools BuildRequires: python%{python3_pkgversion}-Cython @@ -329,6 +330,7 @@ BuildRequires: lz4-devel >= 1.7 %if 0%{with make_check} %if 0%{?fedora} || 0%{?rhel} BuildRequires: golang-github-prometheus +BuildRequires: jsonnet BuildRequires: libtool-ltdl-devel BuildRequires: xmlsec1 BuildRequires: xmlsec1-devel @@ -346,6 +348,7 @@ BuildRequires: python%{python3_pkgversion}-pyOpenSSL %endif %if 0%{?suse_version} BuildRequires: golang-github-prometheus-prometheus +BuildRequires: jsonnet BuildRequires: libxmlsec1-1 BuildRequires: libxmlsec1-nss1 BuildRequires: libxmlsec1-openssl1 @@ -1205,7 +1208,7 @@ This package provides Ceph default alerts for Prometheus. # common ################################################################################# %prep -%autosetup -p1 -n ceph-16.2.5 +%autosetup -p1 -n ceph-16.2.6 %build # LTO can be enabled as soon as the following GCC bug is fixed: @@ -1335,6 +1338,9 @@ ${CMAKE} .. \ -DWITH_SYSTEM_PMDK:BOOL=ON \ %endif -DBOOST_J=$CEPH_SMP_NCPUS \ +%if 0%{?rhel} + -DWITH_FMT_HEADER_ONLY:BOOL=ON \ +%endif -DWITH_GRAFANA=ON %if %{with cmake_verbose_logging} @@ -1990,9 +1996,8 @@ fi %endif %postun immutable-object-cache -test -n "$FIRST_ARG" || FIRST_ARG=$1 %systemd_postun ceph-immutable-object-cache@\*.service ceph-immutable-object-cache.target -if [ $FIRST_ARG -ge 1 ] ; then +if [ $1 -ge 1 ] ; then # Restart on upgrade, but only if "CEPH_AUTO_RESTART_ON_UPGRADE" is set to # "yes". In any case: if units are not running, do not touch them. SYSCONF_CEPH=%{_sysconfdir}/sysconfig/ceph diff --git a/ceph/ceph.spec.in b/ceph/ceph.spec.in index 5e2ffec7d..14fdadaed 100644 --- a/ceph/ceph.spec.in +++ b/ceph/ceph.spec.in @@ -49,6 +49,8 @@ %bcond_without lttng %bcond_without libradosstriper %bcond_without ocf +%global luarocks_package_name luarocks +%bcond_without lua_packages %global _remote_tarball_prefix https://download.ceph.com/tarballs/ %endif %if 0%{?suse_version} @@ -73,6 +75,21 @@ %if ! %{defined _fillupdir} %global _fillupdir /var/adm/fillup-templates %endif +#luarocks +%if 0%{?is_opensuse} +# openSUSE +%bcond_without lua_packages +%if 0%{?sle_version} +# openSUSE Leap +%global luarocks_package_name lua53-luarocks +%else +# openSUSE Tumbleweed +%global luarocks_package_name lua54-luarocks +%endif +%else +# SLE +%bcond_with lua_packages +%endif %endif %bcond_with seastar %bcond_with jaeger @@ -96,19 +113,6 @@ %endif %endif -%if 0%{?suse_version} -%if !0%{?is_opensuse} -# SLE does not support luarocks -%bcond_with lua_packages -%else -%global luarocks_package_name lua53-luarocks -%bcond_without lua_packages -%endif -%else -%global luarocks_package_name luarocks -%bcond_without lua_packages -%endif - %{!?_udevrulesdir: %global _udevrulesdir /lib/udev/rules.d} %{!?tmpfiles_create: %global tmpfiles_create systemd-tmpfiles --create} %{!?python3_pkgversion: %global python3_pkgversion 3} @@ -168,7 +172,6 @@ BuildRequires: gcc-toolset-9-gcc-c++ >= 9.2.1-2.3 %else BuildRequires: gcc-c++ %endif -BuildRequires: gdbm %if 0%{with tcmalloc} # libprofiler did not build on ppc64le until 2.7.90 %if 0%{?fedora} || 0%{?rhel} >= 8 @@ -292,7 +295,6 @@ BuildRequires: libbz2-devel BuildRequires: mozilla-nss-devel BuildRequires: keyutils-devel BuildRequires: libopenssl-devel -BuildRequires: lsb-release BuildRequires: openldap2-devel #BuildRequires: krb5 #BuildRequires: krb5-devel @@ -317,7 +319,6 @@ BuildRequires: openldap-devel #BuildRequires: krb5-devel BuildRequires: openssl-devel BuildRequires: CUnit-devel -BuildRequires: redhat-lsb-core BuildRequires: python%{python3_pkgversion}-devel BuildRequires: python%{python3_pkgversion}-setuptools BuildRequires: python%{python3_pkgversion}-Cython @@ -329,6 +330,7 @@ BuildRequires: lz4-devel >= 1.7 %if 0%{with make_check} %if 0%{?fedora} || 0%{?rhel} BuildRequires: golang-github-prometheus +BuildRequires: jsonnet BuildRequires: libtool-ltdl-devel BuildRequires: xmlsec1 BuildRequires: xmlsec1-devel @@ -346,6 +348,7 @@ BuildRequires: python%{python3_pkgversion}-pyOpenSSL %endif %if 0%{?suse_version} BuildRequires: golang-github-prometheus-prometheus +BuildRequires: jsonnet BuildRequires: libxmlsec1-1 BuildRequires: libxmlsec1-nss1 BuildRequires: libxmlsec1-openssl1 @@ -1335,6 +1338,9 @@ ${CMAKE} .. \ -DWITH_SYSTEM_PMDK:BOOL=ON \ %endif -DBOOST_J=$CEPH_SMP_NCPUS \ +%if 0%{?rhel} + -DWITH_FMT_HEADER_ONLY:BOOL=ON \ +%endif -DWITH_GRAFANA=ON %if %{with cmake_verbose_logging} @@ -1990,9 +1996,8 @@ fi %endif %postun immutable-object-cache -test -n "$FIRST_ARG" || FIRST_ARG=$1 %systemd_postun ceph-immutable-object-cache@\*.service ceph-immutable-object-cache.target -if [ $FIRST_ARG -ge 1 ] ; then +if [ $1 -ge 1 ] ; then # Restart on upgrade, but only if "CEPH_AUTO_RESTART_ON_UPGRADE" is set to # "yes". In any case: if units are not running, do not touch them. SYSCONF_CEPH=%{_sysconfdir}/sysconfig/ceph diff --git a/ceph/changelog.upstream b/ceph/changelog.upstream index 1bf5eab85..51595d76d 100644 --- a/ceph/changelog.upstream +++ b/ceph/changelog.upstream @@ -1,7 +1,8 @@ -ceph (16.2.5-1focal) focal; urgency=medium +ceph (16.2.6-1) stable; urgency=medium + * New upstream release - -- Jenkins Build Slave User Thu, 08 Jul 2021 14:16:59 +0000 + -- Ceph Release Team Thu, 16 Sep 2021 14:27:16 +0000 ceph (16.2.5-1) stable; urgency=medium diff --git a/ceph/cmake/modules/BuildBoost.cmake b/ceph/cmake/modules/BuildBoost.cmake index ba86ecaa6..468ae419c 100644 --- a/ceph/cmake/modules/BuildBoost.cmake +++ b/ceph/cmake/modules/BuildBoost.cmake @@ -155,7 +155,7 @@ function(do_build_boost version) set(boost_sha256 4eb3b8d442b426dc35346235c8733b5ae35ba431690e38c6a8263dce9fcbb402) string(REPLACE "." "_" boost_version_underscore ${boost_version} ) set(boost_url - https://dl.bintray.com/boostorg/release/${boost_version}/source/boost_${boost_version_underscore}.tar.bz2) + https://boostorg.jfrog.io/artifactory/main/release/${boost_version}/source/boost_${boost_version_underscore}.tar.bz2) if(CMAKE_VERSION VERSION_GREATER 3.7) set(boost_url "${boost_url} http://downloads.sourceforge.net/project/boost/boost/${boost_version}/boost_${boost_version_underscore}.tar.bz2") diff --git a/ceph/cmake/modules/CephChecks.cmake b/ceph/cmake/modules/CephChecks.cmake index ca86dcbc7..6a9483f1a 100644 --- a/ceph/cmake/modules/CephChecks.cmake +++ b/ceph/cmake/modules/CephChecks.cmake @@ -56,7 +56,7 @@ endif() CHECK_INCLUDE_FILES("valgrind/helgrind.h" HAVE_VALGRIND_HELGRIND_H) include(CheckTypeSize) -set(CMAKE_EXTRA_INCLUDE_FILES "linux/types.h") +set(CMAKE_EXTRA_INCLUDE_FILES "linux/types.h" "netinet/in.h") CHECK_TYPE_SIZE(__u8 __U8) CHECK_TYPE_SIZE(__u16 __U16) CHECK_TYPE_SIZE(__u32 __U32) @@ -65,6 +65,7 @@ CHECK_TYPE_SIZE(__s8 __S8) CHECK_TYPE_SIZE(__s16 __S16) CHECK_TYPE_SIZE(__s32 __S32) CHECK_TYPE_SIZE(__s64 __S64) +CHECK_TYPE_SIZE(in_addr_t IN_ADDR_T) unset(CMAKE_EXTRA_INCLUDE_FILES) include(CheckSymbolExists) diff --git a/ceph/cmake/modules/Findfmt.cmake b/ceph/cmake/modules/Findfmt.cmake index 747d924e9..734c2b057 100644 --- a/ceph/cmake/modules/Findfmt.cmake +++ b/ceph/cmake/modules/Findfmt.cmake @@ -35,9 +35,27 @@ mark_as_advanced( fmt_VERSION_STRING) if(fmt_FOUND AND NOT (TARGET fmt::fmt)) - add_library(fmt::fmt UNKNOWN IMPORTED) - set_target_properties(fmt::fmt PROPERTIES + add_library(fmt-header-only INTERFACE) + set_target_properties(fmt-header-only PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${fmt_INCLUDE_DIR}" + INTERFACE_COMPILE_DEFINITIONS FMT_HEADER_ONLY=1 + INTERFACE_COMPILE_FEATURES cxx_std_11) + + add_library(fmt UNKNOWN IMPORTED GLOBAL) + set_target_properties(fmt PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${fmt_INCLUDE_DIR}" + INTERFACE_COMPILE_FEATURES cxx_std_11 IMPORTED_LINK_INTERFACE_LANGUAGES "CXX" IMPORTED_LOCATION "${fmt_LIBRARY}") + + if(WITH_FMT_HEADER_ONLY) + # please note, this is different from how upstream defines fmt::fmt. + # in order to force 3rd party libraries to link against fmt-header-only if + # WITH_FMT_HEADER_ONLY is ON, we have to point fmt::fmt to fmt-header-only + # in this case. + add_library(fmt::fmt ALIAS fmt-header-only) + else() + add_library(fmt::fmt ALIAS fmt) + endif() + endif() diff --git a/ceph/debian/control b/ceph/debian/control index a319e5a82..753c5266d 100644 --- a/ceph/debian/control +++ b/ceph/debian/control @@ -12,19 +12,18 @@ Build-Depends: automake, cmake (>= 3.10.2), cpio, cryptsetup-bin | cryptsetup, - cython, cython3, - debhelper (>= 9), + debhelper (>= 10), default-jdk, dh-exec, dh-python, - dh-systemd, # Jaeger flex, git, gperf, g++ (>= 7), javahelper, -# Make-Check jq, + jq , + jsonnet , junit4, libaio-dev, libbabeltrace-ctf-dev, @@ -74,7 +73,6 @@ Build-Depends: automake, # Make-Check libxmlsec1-openssl, # Make-Check libxmlsec1-dev, # Crimson libyaml-cpp-dev, - lsb-release, # Jaeger nlohmann-json-dev | nlohmann-json3-dev, parted, patch, diff --git a/ceph/doc/_ext/ceph_commands.py b/ceph/doc/_ext/ceph_commands.py index 58daba8f1..f1d7ee606 100644 --- a/ceph/doc/_ext/ceph_commands.py +++ b/ceph/doc/_ext/ceph_commands.py @@ -254,6 +254,7 @@ class CephMgrCommands(Directive): 'jsonpatch', 'rook.rook_client', 'rook.rook_client.ceph', + 'rook.rook_client._helper', 'cherrypy=3.2.3'] # make restful happy diff --git a/ceph/doc/ceph-volume/index.rst b/ceph/doc/ceph-volume/index.rst index a9c18abb7..9271bc2a0 100644 --- a/ceph/doc/ceph-volume/index.rst +++ b/ceph/doc/ceph-volume/index.rst @@ -76,6 +76,9 @@ and ``ceph-disk`` is fully disabled. Encryption is fully supported. lvm/systemd lvm/list lvm/zap + lvm/migrate + lvm/newdb + lvm/newwal simple/index simple/activate simple/scan diff --git a/ceph/doc/ceph-volume/lvm/index.rst b/ceph/doc/ceph-volume/lvm/index.rst index 9a2191fb5..962e51a51 100644 --- a/ceph/doc/ceph-volume/lvm/index.rst +++ b/ceph/doc/ceph-volume/lvm/index.rst @@ -15,6 +15,12 @@ Implements the functionality needed to deploy OSDs from the ``lvm`` subcommand: * :ref:`ceph-volume-lvm-list` +* :ref:`ceph-volume-lvm-migrate` + +* :ref:`ceph-volume-lvm-newdb` + +* :ref:`ceph-volume-lvm-newwal` + .. not yet implemented .. * :ref:`ceph-volume-lvm-scan` diff --git a/ceph/doc/ceph-volume/lvm/migrate.rst b/ceph/doc/ceph-volume/lvm/migrate.rst new file mode 100644 index 000000000..983d2e797 --- /dev/null +++ b/ceph/doc/ceph-volume/lvm/migrate.rst @@ -0,0 +1,47 @@ +.. _ceph-volume-lvm-migrate: + +``migrate`` +=========== + +Moves BlueFS data from source volume(s) to the target one, source volumes +(except the main, i.e. data or block one) are removed on success. + +LVM volumes are permitted for Target only, both already attached or new one. + +In the latter case it is attached to the OSD replacing one of the source +devices. + +Following replacement rules apply (in the order of precedence, stop +on the first match): + + - if source list has DB volume - target device replaces it. + - if source list has WAL volume - target device replaces it. + - if source list has slow volume only - operation is not permitted, + requires explicit allocation via new-db/new-wal command. + +Moves BlueFS data from main device to LV already attached as DB:: + + ceph-volume lvm migrate --osd-id 1 --osd-fsid --from data --target vgname/db + +Moves BlueFS data from shared main device to LV which will be attached as a +new DB:: + + ceph-volume lvm migrate --osd-id 1 --osd-fsid --from data --target vgname/new_db + +Moves BlueFS data from DB device to new LV, DB is replaced:: + + ceph-volume lvm migrate --osd-id 1 --osd-fsid --from db --target vgname/new_db + +Moves BlueFS data from main and DB devices to new LV, DB is replaced:: + + ceph-volume lvm migrate --osd-id 1 --osd-fsid --from data db --target vgname/new_db + +Moves BlueFS data from main, DB and WAL devices to new LV, WAL is removed and +DB is replaced:: + + ceph-volume lvm migrate --osd-id 1 --osd-fsid --from data db wal --target vgname/new_db + +Moves BlueFS data from main, DB and WAL devices to main device, WAL and DB are +removed:: + + ceph-volume lvm migrate --osd-id 1 --osd-fsid --from db wal --target vgname/data diff --git a/ceph/doc/ceph-volume/lvm/newdb.rst b/ceph/doc/ceph-volume/lvm/newdb.rst new file mode 100644 index 000000000..dcc87fc8a --- /dev/null +++ b/ceph/doc/ceph-volume/lvm/newdb.rst @@ -0,0 +1,11 @@ +.. _ceph-volume-lvm-newdb: + +``new-db`` +=========== + +Attaches the given logical volume to OSD as a DB. +Logical volume name format is vg/lv. Fails if OSD has already got attached DB. + +Attach vgname/lvname as a DB volume to OSD 1:: + + ceph-volume lvm new-db --osd-id 1 --osd-fsid 55BD4219-16A7-4037-BC20-0F158EFCC83D --target vgname/new_db diff --git a/ceph/doc/ceph-volume/lvm/newwal.rst b/ceph/doc/ceph-volume/lvm/newwal.rst new file mode 100644 index 000000000..05f87fff6 --- /dev/null +++ b/ceph/doc/ceph-volume/lvm/newwal.rst @@ -0,0 +1,11 @@ +.. _ceph-volume-lvm-newwal: + +``new-wal`` +=========== + +Attaches the given logical volume to the given OSD as a WAL volume. +Logical volume format is vg/lv. Fails if OSD has already got attached DB. + +Attach vgname/lvname as a WAL volume to OSD 1:: + + ceph-volume lvm new-wal --osd-id 1 --osd-fsid 55BD4219-16A7-4037-BC20-0F158EFCC83D --target vgname/new_wal diff --git a/ceph/doc/cephadm/client-setup.rst b/ceph/doc/cephadm/client-setup.rst index 3efc1cc11..f98ba798b 100644 --- a/ceph/doc/cephadm/client-setup.rst +++ b/ceph/doc/cephadm/client-setup.rst @@ -1,40 +1,45 @@ ======================= Basic Ceph Client Setup ======================= -Client machines need some basic configuration in order to interact with -a cluster. This document describes how to configure a client machine -for cluster interaction. +Client machines require some basic configuration to interact with +Ceph clusters. This section describes how to configure a client machine +so that it can interact with a Ceph cluster. -.. note:: Most client machines only need the `ceph-common` package and - its dependencies installed. That will supply the basic `ceph` - and `rados` commands, as well as other commands like - `mount.ceph` and `rbd`. +.. note:: + Most client machines need to install only the `ceph-common` package + and its dependencies. Such a setup supplies the basic `ceph` and + `rados` commands, as well as other commands including `mount.ceph` + and `rbd`. Config File Setup ================= -Client machines can generally get away with a smaller config file than -a full-fledged cluster member. To generate a minimal config file, log -into a host that is already configured as a client or running a cluster -daemon, and then run +Client machines usually require smaller configuration files (here +sometimes called "config files") than do full-fledged cluster members. +To generate a minimal config file, log into a host that has been +configured as a client or that is running a cluster daemon, and then run the following command: -.. code-block:: bash +.. prompt:: bash # - ceph config generate-minimal-conf + ceph config generate-minimal-conf -This will generate a minimal config file that will tell the client how to -reach the Ceph Monitors. The contents of this file should typically be -installed in `/etc/ceph/ceph.conf`. +This command generates a minimal config file that tells the client how +to reach the Ceph monitors. The contents of this file should usually +be installed in ``/etc/ceph/ceph.conf``. Keyring Setup ============= -Most Ceph clusters are run with authentication enabled, and the client will -need keys in order to communicate with cluster machines. To generate a -keyring file with credentials for `client.fs`, log into an extant cluster -member and run +Most Ceph clusters run with authentication enabled. This means that +the client needs keys in order to communicate with the machines in the +cluster. To generate a keyring file with credentials for `client.fs`, +log into an running cluster member and run the following command: -.. code-block:: bash +.. prompt:: bash $ - ceph auth get-or-create client.fs + ceph auth get-or-create client.fs -The resulting output should be put into a keyring file, typically -`/etc/ceph/ceph.keyring`. +The resulting output is directed into a keyring file, typically +``/etc/ceph/ceph.keyring``. + +To gain a broader understanding of client keyring distribution and administration, you should read :ref:`client_keyrings_and_configs`. + +To see an example that explains how to distribute ``ceph.conf`` configuration files to hosts that are tagged with the ``bare_config`` label, you should read the section called "Distributing ceph.conf to hosts tagged with bare_config" in the section called :ref:`etc_ceph_conf_distribution`. diff --git a/ceph/doc/cephadm/host-management.rst b/ceph/doc/cephadm/host-management.rst index bcf626752..621f2a753 100644 --- a/ceph/doc/cephadm/host-management.rst +++ b/ceph/doc/cephadm/host-management.rst @@ -64,48 +64,47 @@ To add each new host to the cluster, perform two steps: Removing Hosts ============== -If the node that want you to remove is running OSDs, make sure you remove the OSDs from the node. +A host can safely be removed from a the cluster once all daemons are removed from it. -To remove a host from a cluster, do the following: +To drain all daemons from a host do the following: -For all Ceph service types, except for ``node-exporter`` and ``crash``, remove -the host from the placement specification file (for example, cluster.yml). -For example, if you are removing the host named host2, remove all occurrences of -``- host2`` from all ``placement:`` sections. +.. prompt:: bash # + + ceph orch host drain ** + +The '_no_schedule' label will be applied to the host. See :ref:`cephadm-special-host-labels` -Update: +All osds on the host will be scheduled to be removed. You can check osd removal progress with the following: -.. code-block:: yaml +.. prompt:: bash # - service_type: rgw - placement: - hosts: - - host1 - - host2 + ceph orch osd rm status -To: +see :ref:`cephadm-osd-removal` for more details about osd removal -.. code-block:: yaml +You can check if there are no deamons left on the host with the following: +.. prompt:: bash # - service_type: rgw - placement: - hosts: - - host1 + ceph orch ps -Remove the host from cephadm's environment: +Once all daemons are removed you can remove the host with the following: .. prompt:: bash # - ceph orch host rm host2 + ceph orch host rm +Offline host removal +-------------------- -If the host is running ``node-exporter`` and crash services, remove them by running -the following command on the host: +If a host is offline and can not be recovered it can still be removed from the cluster with the following: .. prompt:: bash # - cephadm rm-daemon --fsid CLUSTER_ID --name SERVICE_NAME + ceph orch host rm --offline --force + +This can potentially cause data loss as osds will be forcefully purged from the cluster by calling ``osd purge-actual`` for each osd. +Service specs that still contain this host should be manually updated. .. _orchestrator-host-labels: diff --git a/ceph/doc/cephadm/install.rst b/ceph/doc/cephadm/install.rst index 2a4a8887a..25907296c 100644 --- a/ceph/doc/cephadm/install.rst +++ b/ceph/doc/cephadm/install.rst @@ -173,6 +173,11 @@ immediately to know more about ``cephadm bootstrap``, read the list below. Also, you can run ``cephadm bootstrap -h`` to see all of ``cephadm``'s available options. +* By default, Ceph daemons send their log output to stdout/stderr, which is picked + up by the container runtime (docker or podman) and (on most systems) sent to + journald. If you want Ceph to write traditional log files to ``/var/log/ceph/$fsid``, + use ``--log-to-file`` option during bootstrap. + * Larger Ceph clusters perform better when (external to the Ceph cluster) public network traffic is separated from (internal to the Ceph cluster) cluster traffic. The internal cluster traffic handles replication, recovery, diff --git a/ceph/doc/cephadm/mon.rst b/ceph/doc/cephadm/mon.rst index 38ae493f5..e66df6171 100644 --- a/ceph/doc/cephadm/mon.rst +++ b/ceph/doc/cephadm/mon.rst @@ -26,7 +26,11 @@ default to that subnet unless cephadm is instructed to do otherwise. If all of the ceph monitor daemons in your cluster are in the same subnet, manual administration of the ceph monitor daemons is not necessary. ``cephadm`` will automatically add up to five monitors to the subnet, as -needed, as new hosts are added to the cluster. +needed, as new hosts are added to the cluster. + +By default, cephadm will deploy 5 daemons on arbitrary hosts. See +:ref:`orchestrator-cli-placement-spec` for details of specifying +the placement of daemons. Designating a Particular Subnet for Monitors -------------------------------------------- @@ -48,88 +52,84 @@ format (e.g., ``10.1.2.0/24``): Cephadm deploys new monitor daemons only on hosts that have IP addresses in the designated subnet. -Changing the number of monitors from the default ------------------------------------------------- - -If you want to adjust the default of 5 monitors, run this command: +You can also specify two public networks by using a list of networks: .. prompt:: bash # - ceph orch apply mon ** + ceph config set mon public_network *,* -Deploying monitors only to specific hosts ------------------------------------------ - -To deploy monitors on a specific set of hosts, run this command: + For example: .. prompt:: bash # - ceph orch apply mon ** + ceph config set mon public_network 10.1.2.0/24,192.168.0.1/24 - Be sure to include the first (bootstrap) host in this list. -Using Host Labels ------------------ +Deploying Monitors on a Particular Network +------------------------------------------ + +You can explicitly specify the IP address or CIDR network for each monitor and +control where each monitor is placed. To disable automated monitor deployment, +run this command: -You can control which hosts the monitors run on by making use of host labels. -To set the ``mon`` label to the appropriate hosts, run this command: - .. prompt:: bash # - ceph orch host label add ** mon + ceph orch apply mon --unmanaged - To view the current hosts and labels, run this command: + To deploy each additional monitor: .. prompt:: bash # - ceph orch host ls + ceph orch daemon add mon * - For example: + For example, to deploy a second monitor on ``newhost1`` using an IP + address ``10.1.2.123`` and a third monitor on ``newhost2`` in + network ``10.1.2.0/24``, run the following commands: .. prompt:: bash # - ceph orch host label add host1 mon - ceph orch host label add host2 mon - ceph orch host label add host3 mon - ceph orch host ls + ceph orch apply mon --unmanaged + ceph orch daemon add mon newhost1:10.1.2.123 + ceph orch daemon add mon newhost2:10.1.2.0/24 + + Now, enable automatic placement of Daemons + + .. prompt:: bash # - .. code-block:: bash + ceph orch apply mon --placement="newhost1,newhost2,newhost3" --dry-run - HOST ADDR LABELS STATUS - host1 mon - host2 mon - host3 mon - host4 - host5 + See :ref:`orchestrator-cli-placement-spec` for details of specifying + the placement of daemons. - Tell cephadm to deploy monitors based on the label by running this command: + Finally apply this new placement by dropping ``--dry-run`` .. prompt:: bash # - ceph orch apply mon label:mon + ceph orch apply mon --placement="newhost1,newhost2,newhost3" -See also :ref:`host labels `. -Deploying Monitors on a Particular Network ------------------------------------------- +Moving Monitors to a Different Network +-------------------------------------- -You can explicitly specify the IP address or CIDR network for each monitor and -control where each monitor is placed. To disable automated monitor deployment, -run this command: +To move Monitors to a new network, deploy new monitors on the new network and +subsequently remove monitors from the old network. It is not advised to +modify and inject the ``monmap`` manually. + +First, disable the automated placement of daemons: .. prompt:: bash # ceph orch apply mon --unmanaged - To deploy each additional monitor: +To deploy each additional monitor: .. prompt:: bash # - ceph orch daemon add mon * [...]* + ceph orch daemon add mon ** - For example, to deploy a second monitor on ``newhost1`` using an IP - address ``10.1.2.123`` and a third monitor on ``newhost2`` in - network ``10.1.2.0/24``, run the following commands: +For example, to deploy a second monitor on ``newhost1`` using an IP +address ``10.1.2.123`` and a third monitor on ``newhost2`` in +network ``10.1.2.0/24``, run the following commands: .. prompt:: bash # @@ -137,52 +137,35 @@ run this command: ceph orch daemon add mon newhost1:10.1.2.123 ceph orch daemon add mon newhost2:10.1.2.0/24 - .. note:: - The **apply** command can be confusing. For this reason, we recommend using - YAML specifications. - - Each ``ceph orch apply mon`` command supersedes the one before it. - This means that you must use the proper comma-separated list-based - syntax when you want to apply monitors to more than one host. - If you do not use the proper syntax, you will clobber your work - as you go. - - For example: - - .. prompt:: bash # - - ceph orch apply mon host1 - ceph orch apply mon host2 - ceph orch apply mon host3 - - This results in only one host having a monitor applied to it: host 3. - - (The first command creates a monitor on host1. Then the second command - clobbers the monitor on host1 and creates a monitor on host2. Then the - third command clobbers the monitor on host2 and creates a monitor on - host3. In this scenario, at this point, there is a monitor ONLY on - host3.) - - To make certain that a monitor is applied to each of these three hosts, - run a command like this: - - .. prompt:: bash # - - ceph orch apply mon "host1,host2,host3" - - There is another way to apply monitors to multiple hosts: a ``yaml`` file - can be used. Instead of using the "ceph orch apply mon" commands, run a - command of this form: - - .. prompt:: bash # - - ceph orch apply -i file.yaml - - Here is a sample **file.yaml** file:: - - service_type: mon - placement: - hosts: - - host1 - - host2 - - host3 + Subsequently remove monitors from the old network: + + .. prompt:: bash # + + ceph orch daemon rm *mon.* + + Update the ``public_network``: + + .. prompt:: bash # + + ceph config set mon public_network ** + + For example: + + .. prompt:: bash # + + ceph config set mon public_network 10.1.2.0/24 + + Now, enable automatic placement of Daemons + + .. prompt:: bash # + + ceph orch apply mon --placement="newhost1,newhost2,newhost3" --dry-run + + See :ref:`orchestrator-cli-placement-spec` for details of specifying + the placement of daemons. + + Finally apply this new placement by dropping ``--dry-run`` + + .. prompt:: bash # + + ceph orch apply mon --placement="newhost1,newhost2,newhost3" diff --git a/ceph/doc/cephadm/monitoring.rst b/ceph/doc/cephadm/monitoring.rst index 14593b1bb..38f4b4bb4 100644 --- a/ceph/doc/cephadm/monitoring.rst +++ b/ceph/doc/cephadm/monitoring.rst @@ -52,12 +52,6 @@ cluster (which had no monitoring stack) to cephadm management.) To set up monitoring on a Ceph cluster that has no monitoring, follow the steps below: -#. Enable the Prometheus module in the ceph-mgr daemon. This exposes the internal Ceph metrics so that Prometheus can scrape them: - - .. prompt:: bash # - - ceph mgr module enable prometheus - #. Deploy a node-exporter service on every node of the cluster. The node-exporter provides host-level metrics like CPU and memory utilization: .. prompt:: bash # diff --git a/ceph/doc/cephadm/operations.rst b/ceph/doc/cephadm/operations.rst index 56d1146de..45ae1575b 100644 --- a/ceph/doc/cephadm/operations.rst +++ b/ceph/doc/cephadm/operations.rst @@ -2,28 +2,40 @@ Cephadm Operations ================== +.. _watching_cephadm_logs: + Watching cephadm log messages ============================= -Cephadm logs to the ``cephadm`` cluster log channel, meaning you can -monitor progress in realtime with:: +Cephadm writes logs to the ``cephadm`` cluster log channel. You can +monitor Ceph's activity in real time by reading the logs as they fill +up. Run the following command to see the logs in real time: + +.. prompt:: bash # + + ceph -W cephadm + +By default, this command shows info-level events and above. To see +debug-level messages as well as info-level events, run the following +commands: - # ceph -W cephadm +.. prompt:: bash # -By default it will show info-level events and above. To see -debug-level messages too:: + ceph config set mgr mgr/cephadm/log_to_cluster_level debug + ceph -W cephadm --watch-debug - # ceph config set mgr mgr/cephadm/log_to_cluster_level debug - # ceph -W cephadm --watch-debug +.. warning:: -Be careful: the debug messages are very verbose! + The debug messages are very verbose! -You can see recent events with:: +You can see recent events by running the following command: - # ceph log last cephadm +.. prompt:: bash # + + ceph log last cephadm These events are also logged to the ``ceph.cephadm.log`` file on -monitor hosts and to the monitor daemons' stderr. +monitor hosts as well as to the monitor daemons' stderr. .. _cephadm-logs: @@ -31,45 +43,68 @@ monitor hosts and to the monitor daemons' stderr. Ceph daemon logs ================ -Logging to stdout ------------------ +Logging to journald +------------------- + +Ceph daemons traditionally write logs to ``/var/log/ceph``. Ceph daemons log to +journald by default and Ceph logs are captured by the container runtime +environment. They are accessible via ``journalctl``. + +.. note:: Prior to Quincy, ceph daemons logged to stderr. -Traditionally, Ceph daemons have logged to ``/var/log/ceph``. By -default, cephadm daemons log to stderr and the logs are -captured by the container runtime environment. For most systems, by -default, these logs are sent to journald and accessible via -``journalctl``. +Example of logging to journald +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ For example, to view the logs for the daemon ``mon.foo`` for a cluster with ID ``5c5a50ae-272a-455d-99e9-32c6a013e694``, the command would be -something like:: +something like: + +.. prompt:: bash # journalctl -u ceph-5c5a50ae-272a-455d-99e9-32c6a013e694@mon.foo This works well for normal operations when logging levels are low. -To disable logging to stderr:: - - ceph config set global log_to_stderr false - ceph config set global mon_cluster_log_to_stderr false - Logging to files ---------------- -You can also configure Ceph daemons to log to files instead of stderr, -just like they have in the past. When logging to files, Ceph logs appear -in ``/var/log/ceph/``. +You can also configure Ceph daemons to log to files instead of to +journald if you prefer logs to appear in files (as they did in earlier, +pre-cephadm, pre-Octopus versions of Ceph). When Ceph logs to files, +the logs appear in ``/var/log/ceph/``. If you choose to +configure Ceph to log to files instead of to journald, remember to +configure Ceph so that it will not log to journald (the commands for +this are covered below). + +Enabling logging to files +~~~~~~~~~~~~~~~~~~~~~~~~~ + +To enable logging to files, run the following commands: -To enable logging to files:: +.. prompt:: bash # ceph config set global log_to_file true ceph config set global mon_cluster_log_to_file true -We recommend disabling logging to stderr (see above) or else everything -will be logged twice:: +Disabling logging to journald +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you choose to log to files, we recommend disabling logging to journald or else +everything will be logged twice. Run the following commands to disable logging +to stderr: + +.. prompt:: bash # ceph config set global log_to_stderr false ceph config set global mon_cluster_log_to_stderr false + ceph config set global log_to_journald false + ceph config set global mon_cluster_log_to_journald false + +.. note:: You can change the default by passing --log-to-file during + bootstrapping a new cluster. + +Modifying the log retention schedule +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ By default, cephadm sets up log rotation on each host to rotate these files. You can configure the logging retention schedule by modifying @@ -79,12 +114,13 @@ files. You can configure the logging retention schedule by modifying Data location ============= -Cephadm daemon data and logs in slightly different locations than older -versions of ceph: +Cephadm stores daemon data and logs in different locations than did +older, pre-cephadm (pre Octopus) versions of ceph: -* ``/var/log/ceph/`` contains all cluster logs. Note - that by default cephadm logs via stderr and the container runtime, - so these logs are normally not present. +* ``/var/log/ceph/`` contains all cluster logs. By + default, cephadm logs via stderr and the container runtime. These + logs will not exist unless you have enabled logging to files as + described in `cephadm-logs`_. * ``/var/lib/ceph/`` contains all cluster daemon data (besides logs). * ``/var/lib/ceph//`` contains all data for @@ -98,58 +134,69 @@ versions of ceph: Disk usage ---------- -Because a few Ceph daemons may store a significant amount of data in -``/var/lib/ceph`` (notably, the monitors and prometheus), we recommend -moving this directory to its own disk, partition, or logical volume so -that it does not fill up the root file system. +Because a few Ceph daemons (notably, the monitors and prometheus) store a +large amount of data in ``/var/lib/ceph`` , we recommend moving this +directory to its own disk, partition, or logical volume so that it does not +fill up the root file system. Health checks ============= -The cephadm module provides additional healthchecks to supplement the default healthchecks -provided by the Cluster. These additional healthchecks fall into two categories; +The cephadm module provides additional health checks to supplement the +default health checks provided by the Cluster. These additional health +checks fall into two categories: -- **cephadm operations**: Healthchecks in this category are always executed when the cephadm module is active. -- **cluster configuration**: These healthchecks are *optional*, and focus on the configuration of the hosts in - the cluster +- **cephadm operations**: Health checks in this category are always + executed when the cephadm module is active. +- **cluster configuration**: These health checks are *optional*, and + focus on the configuration of the hosts in the cluster. CEPHADM Operations ------------------ CEPHADM_PAUSED -^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~ -Cephadm background work has been paused with ``ceph orch pause``. Cephadm -continues to perform passive monitoring activities (like checking -host and daemon status), but it will not make any changes (like deploying -or removing daemons). +This indicates that cephadm background work has been paused with +``ceph orch pause``. Cephadm continues to perform passive monitoring +activities (like checking host and daemon status), but it will not +make any changes (like deploying or removing daemons). -Resume cephadm work with:: +Resume cephadm work by running the following command: + +.. prompt:: bash # ceph orch resume .. _cephadm-stray-host: CEPHADM_STRAY_HOST -^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~ + +This indicates that one or more hosts have Ceph daemons that are +running, but are not registered as hosts managed by *cephadm*. This +means that those services cannot currently be managed by cephadm +(e.g., restarted, upgraded, included in `ceph orch ps`). -One or more hosts have running Ceph daemons but are not registered as -hosts managed by *cephadm*. This means that those services cannot -currently be managed by cephadm (e.g., restarted, upgraded, included -in `ceph orch ps`). +You can manage the host(s) by running the following command: -You can manage the host(s) with:: +.. prompt:: bash # ceph orch host add ** -Note that you may need to configure SSH access to the remote host -before this will work. +.. note:: + + You might need to configure SSH access to the remote host + before this will work. Alternatively, you can manually connect to the host and ensure that services on that host are removed or migrated to a host that is managed by *cephadm*. -You can also disable this warning entirely with:: +This warning can be disabled entirely by running the following +command: + +.. prompt:: bash # ceph config set mgr mgr/cephadm/warn_on_stray_hosts false @@ -157,7 +204,7 @@ See :ref:`cephadm-fqdn` for more information about host names and domain names. CEPHADM_STRAY_DAEMON -^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~ One or more Ceph daemons are running but not are not managed by *cephadm*. This may be because they were deployed using a different @@ -170,12 +217,14 @@ by cephadm; see :ref:`cephadm-adoption`. For stateless daemons, it is usually easiest to provision a new daemon with the ``ceph orch apply`` command and then stop the unmanaged daemon. -This warning can be disabled entirely with:: +This warning can be disabled entirely by running the following command: + +.. prompt:: bash # ceph config set mgr mgr/cephadm/warn_on_stray_daemons false CEPHADM_HOST_CHECK_FAILED -^^^^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~~~~ One or more hosts have failed the basic cephadm host check, which verifies that (1) the host is reachable and cephadm can be executed there, and (2) @@ -183,58 +232,80 @@ that the host satisfies basic prerequisites, like a working container runtime (podman or docker) and working time synchronization. If this test fails, cephadm will no be able to manage services on that host. -You can manually run this check with:: +You can manually run this check by running the following command: + +.. prompt:: bash # ceph cephadm check-host ** -You can remove a broken host from management with:: +You can remove a broken host from management by running the following command: + +.. prompt:: bash # ceph orch host rm ** -You can disable this health warning with:: +You can disable this health warning by running the following command: + +.. prompt:: bash # ceph config set mgr mgr/cephadm/warn_on_failed_host_check false Cluster Configuration Checks ---------------------------- -Cephadm periodically scans each of the hosts in the cluster, to understand the state -of the OS, disks, NICs etc. These facts can then be analysed for consistency across the hosts -in the cluster to identify any configuration anomalies. +Cephadm periodically scans each of the hosts in the cluster in order +to understand the state of the OS, disks, NICs etc. These facts can +then be analysed for consistency across the hosts in the cluster to +identify any configuration anomalies. + +Enabling Cluster Configuration Checks +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The configuration checks are an **optional** feature, enabled by the following command -:: +The configuration checks are an **optional** feature, and are enabled +by running the following command: + +.. prompt:: bash # ceph config set mgr mgr/cephadm/config_checks_enabled true -The configuration checks are triggered after each host scan (1m). The cephadm log entries will -show the current state and outcome of the configuration checks as follows; +States Returned by Cluster Configuration Checks +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The configuration checks are triggered after each host scan (1m). The +cephadm log entries will show the current state and outcome of the +configuration checks as follows: -Disabled state (config_checks_enabled false) -:: +Disabled state (config_checks_enabled false): + +.. code-block:: bash ALL cephadm checks are disabled, use 'ceph config set mgr mgr/cephadm/config_checks_enabled true' to enable -Enabled state (config_checks_enabled true) -:: +Enabled state (config_checks_enabled true): + +.. code-block:: bash CEPHADM 8/8 checks enabled and executed (0 bypassed, 0 disabled). No issues detected -The configuration checks themselves are managed through several cephadm sub-commands. +Managing Configuration Checks (subcommands) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To determine whether the configuration checks are enabled, you can use the following command -:: +The configuration checks themselves are managed through several cephadm subcommands. + +To determine whether the configuration checks are enabled, run the following command: + +.. prompt:: bash # ceph cephadm config-check status -This command will return the status of the configuration checker as either "Enabled" or "Disabled". +This command returns the status of the configuration checker as either "Enabled" or "Disabled". + +To list all the configuration checks and their current states, run the following command: -Listing all the configuration checks and their current state -:: +.. code-block:: console - ceph cephadm config-check ls + # ceph cephadm config-check ls - e.g. NAME HEALTHCHECK STATUS DESCRIPTION kernel_security CEPHADM_CHECK_KERNEL_LSM enabled checks SELINUX/Apparmor profiles are consistent across cluster hosts os_subscription CEPHADM_CHECK_SUBSCRIPTION enabled checks subscription states are consistent for all cluster hosts @@ -245,128 +316,191 @@ Listing all the configuration checks and their current state ceph_release CEPHADM_CHECK_CEPH_RELEASE enabled check for Ceph version consistency - ceph daemons should be on the same release (unless upgrade is active) kernel_version CEPHADM_CHECK_KERNEL_VERSION enabled checks that the MAJ.MIN of the kernel on Ceph hosts is consistent -The name of each configuration check, can then be used to enable or disable a specific check. -:: +The name of each configuration check can be used to enable or disable a specific check by running a command of the following form: +: + +.. prompt:: bash # ceph cephadm config-check disable - eg. +For example: + +.. prompt:: bash # + ceph cephadm config-check disable kernel_security CEPHADM_CHECK_KERNEL_LSM -^^^^^^^^^^^^^^^^^^^^^^^^ -Each host within the cluster is expected to operate within the same Linux Security Module (LSM) state. For example, -if the majority of the hosts are running with SELINUX in enforcing mode, any host not running in this mode -would be flagged as an anomaly and a healtcheck (WARNING) state raised. +~~~~~~~~~~~~~~~~~~~~~~~~ +Each host within the cluster is expected to operate within the same Linux +Security Module (LSM) state. For example, if the majority of the hosts are +running with SELINUX in enforcing mode, any host not running in this mode is +flagged as an anomaly and a healtcheck (WARNING) state raised. CEPHADM_CHECK_SUBSCRIPTION -^^^^^^^^^^^^^^^^^^^^^^^^^^ -This check relates to the status of vendor subscription. This check is only performed for hosts using RHEL, but helps -to confirm that all your hosts are covered by an active subscription so patches and updates -are available. +~~~~~~~~~~~~~~~~~~~~~~~~~~ +This check relates to the status of vendor subscription. This check is +performed only for hosts using RHEL, but helps to confirm that all hosts are +covered by an active subscription, which ensures that patches and updates are +available. CEPHADM_CHECK_PUBLIC_MEMBERSHIP -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -All members of the cluster should have NICs configured on at least one of the public network subnets. Hosts -that are not on the public network will rely on routing which may affect performance +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +All members of the cluster should have NICs configured on at least one of the +public network subnets. Hosts that are not on the public network will rely on +routing, which may affect performance. CEPHADM_CHECK_MTU -^^^^^^^^^^^^^^^^^ -The MTU of the NICs on OSDs can be a key factor in consistent performance. This check examines hosts -that are running OSD services to ensure that the MTU is configured consistently within the cluster. This is -determined by establishing the MTU setting that the majority of hosts are using, with any anomalies being -resulting in a Ceph healthcheck. +~~~~~~~~~~~~~~~~~ +The MTU of the NICs on OSDs can be a key factor in consistent performance. This +check examines hosts that are running OSD services to ensure that the MTU is +configured consistently within the cluster. This is determined by establishing +the MTU setting that the majority of hosts is using. Any anomalies result in a +Ceph health check. CEPHADM_CHECK_LINKSPEED -^^^^^^^^^^^^^^^^^^^^^^^ -Similar to the MTU check, linkspeed consistency is also a factor in consistent cluster performance. -This check determines the linkspeed shared by the majority of "OSD hosts", resulting in a healthcheck for -any hosts that are set at a lower linkspeed rate. +~~~~~~~~~~~~~~~~~~~~~~~ +This check is similar to the MTU check. Linkspeed consistency is a factor in +consistent cluster performance, just as the MTU of the NICs on the OSDs is. +This check determines the linkspeed shared by the majority of OSD hosts, and a +health check is run for any hosts that are set at a lower linkspeed rate. CEPHADM_CHECK_NETWORK_MISSING -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The public_network and cluster_network settings support subnet definitions for IPv4 and IPv6. If these -settings are not found on any host in the cluster a healthcheck is raised. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The `public_network` and `cluster_network` settings support subnet definitions +for IPv4 and IPv6. If these settings are not found on any host in the cluster, +a health check is raised. CEPHADM_CHECK_CEPH_RELEASE -^^^^^^^^^^^^^^^^^^^^^^^^^^ -Under normal operations, the ceph cluster should be running daemons under the same ceph release (i.e. all -pacific). This check looks at the active release for each daemon, and reports any anomalies as a -healthcheck. *This check is bypassed if an upgrade process is active within the cluster.* +~~~~~~~~~~~~~~~~~~~~~~~~~~ +Under normal operations, the Ceph cluster runs daemons under the same ceph +release (that is, the Ceph cluster runs all daemons under (for example) +Octopus). This check determines the active release for each daemon, and +reports any anomalies as a healthcheck. *This check is bypassed if an upgrade +process is active within the cluster.* CEPHADM_CHECK_KERNEL_VERSION -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The OS kernel version (maj.min) is checked for consistency across the hosts. Once again, the -majority of the hosts is used as the basis of identifying anomalies. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The OS kernel version (maj.min) is checked for consistency across the hosts. +The kernel version of the majority of the hosts is used as the basis for +identifying anomalies. + +.. _client_keyrings_and_configs: Client keyrings and configs =========================== -Cephadm can distribute copies of the ``ceph.conf`` and client keyring -files to hosts. For example, it is usually a good idea to store a -copy of the config and ``client.admin`` keyring on any hosts that will -be used to administer the cluster via the CLI. By default, cephadm will do -this for any nodes with the ``_admin`` label (which normally includes the bootstrap -host). +Cephadm can distribute copies of the ``ceph.conf`` file and client keyring +files to hosts. It is usually a good idea to store a copy of the config and +``client.admin`` keyring on any host used to administer the cluster via the +CLI. By default, cephadm does this for any nodes that have the ``_admin`` +label (which normally includes the bootstrap host). When a client keyring is placed under management, cephadm will: - - build a list of target hosts based on the specified placement spec (see :ref:`orchestrator-cli-placement-spec`) + - build a list of target hosts based on the specified placement spec (see + :ref:`orchestrator-cli-placement-spec`) - store a copy of the ``/etc/ceph/ceph.conf`` file on the specified host(s) - store a copy of the keyring file on the specified host(s) - update the ``ceph.conf`` file as needed (e.g., due to a change in the cluster monitors) - - update the keyring file if the entity's key is changed (e.g., via ``ceph auth ...`` commands) - - ensure the keyring file has the specified ownership and mode + - update the keyring file if the entity's key is changed (e.g., via ``ceph + auth ...`` commands) + - ensure that the keyring file has the specified ownership and specified mode - remove the keyring file when client keyring management is disabled - - remove the keyring file from old hosts if the keyring placement spec is updated (as needed) + - remove the keyring file from old hosts if the keyring placement spec is + updated (as needed) -To view which client keyrings are currently under management:: +Listing Client Keyrings +----------------------- + +To see the list of client keyrings are currently under management, run the following command: + +.. prompt:: bash # ceph orch client-keyring ls -To place a keyring under management:: +Putting a Keyring Under Management +---------------------------------- + +To put a keyring under management, run a command of the following form: + +.. prompt:: bash # ceph orch client-keyring set [--mode=] [--owner=.] [--path=] -- By default, the *path* will be ``/etc/ceph/client.{entity}.keyring``, which is where - Ceph looks by default. Be careful specifying alternate locations as existing files - may be overwritten. +- By default, the *path* is ``/etc/ceph/client.{entity}.keyring``, which is + where Ceph looks by default. Be careful when specifying alternate locations, + as existing files may be overwritten. - A placement of ``*`` (all hosts) is common. - The mode defaults to ``0600`` and ownership to ``0:0`` (user root, group root). -For example, to create and deploy a ``client.rbd`` key to hosts with the ``rbd-client`` label and group readable by uid/gid 107 (qemu),:: +For example, to create a ``client.rbd`` key and deploy it to hosts with the +``rbd-client`` label and make it group readable by uid/gid 107 (qemu), run the +following commands: + +.. prompt:: bash # ceph auth get-or-create-key client.rbd mon 'profile rbd' mgr 'profile rbd' osd 'profile rbd pool=my_rbd_pool' ceph orch client-keyring set client.rbd label:rbd-client --owner 107:107 --mode 640 -The resulting keyring file is:: +The resulting keyring file is: + +.. code-block:: console -rw-r-----. 1 qemu qemu 156 Apr 21 08:47 /etc/ceph/client.client.rbd.keyring -To disable management of a keyring file:: +Disabling Management of a Keyring File +-------------------------------------- + +To disable management of a keyring file, run a command of the following form: + +.. prompt:: bash # ceph orch client-keyring rm -Note that this will delete any keyring files for this entity that were previously written -to cluster nodes. +.. note:: + + This deletes any keyring files for this entity that were previously written + to cluster nodes. +.. _etc_ceph_conf_distribution: /etc/ceph/ceph.conf =================== -It may also be useful to distribute ``ceph.conf`` files to hosts without an associated -client keyring file. By default, cephadm only deploys a ``ceph.conf`` file to hosts where a client keyring -is also distributed (see above). To write config files to hosts without client keyrings:: +Distributing ceph.conf to hosts that have no keyrings +----------------------------------------------------- + +It might be useful to distribute ``ceph.conf`` files to hosts without an +associated client keyring file. By default, cephadm deploys only a +``ceph.conf`` file to hosts where a client keyring is also distributed (see +above). To write config files to hosts without client keyrings, run the +following command: + +.. prompt:: bash # ceph config set mgr mgr/cephadm/manage_etc_ceph_ceph_conf true -By default, the configs are written to all hosts (i.e., those listed -by ``ceph orch host ls``). To specify which hosts get a ``ceph.conf``:: +Using Placement Specs to specify which hosts get keyrings +--------------------------------------------------------- + +By default, the configs are written to all hosts (i.e., those listed by ``ceph +orch host ls``). To specify which hosts get a ``ceph.conf``, run a command of +the following form: + +.. prompt:: bash # + + ceph config set mgr mgr/cephadm/manage_etc_ceph_ceph_conf_hosts + +For example, to distribute configs to hosts with the ``bare_config`` label, run +the following command: + +Distributing ceph.conf to hosts tagged with bare_config +------------------------------------------------------- - ceph config set mgr mgr/cephadm/manage_etc_ceph_ceph_conf_hosts +For example, to distribute configs to hosts with the ``bare_config`` label, run the following command: -For example, to distribute configs to hosts with the ``bare_config`` label,:: +.. prompt:: bash # - ceph config set mgr mgr/cephadm/manage_etc_ceph_ceph_conf_hosts label:bare_config + ceph config set mgr mgr/cephadm/manage_etc_ceph_ceph_conf_hosts label:bare_config (See :ref:`orchestrator-cli-placement-spec` for more information about placement specs.) diff --git a/ceph/doc/cephadm/osd.rst b/ceph/doc/cephadm/osd.rst index 5c01d038f..f0bf47cfe 100644 --- a/ceph/doc/cephadm/osd.rst +++ b/ceph/doc/cephadm/osd.rst @@ -7,7 +7,7 @@ OSD Service List Devices ============ -``ceph-volume`` scans each cluster in the host from time to time in order +``ceph-volume`` scans each host in the cluster from time to time in order to determine which devices are present and whether they are eligible to be used as OSDs. @@ -211,6 +211,7 @@ If you want to avoid this behavior (disable automatic creation of OSD on availab * For cephadm, see also :ref:`cephadm-spec-unmanaged`. +.. _cephadm-osd-removal: Remove an OSD ============= @@ -347,7 +348,7 @@ zap`` on the remote host. .. prompt:: bash # - orch device zap + ceph orch device zap Example command: diff --git a/ceph/doc/cephadm/rgw.rst b/ceph/doc/cephadm/rgw.rst index 8329ee27b..3283fdbdf 100644 --- a/ceph/doc/cephadm/rgw.rst +++ b/ceph/doc/cephadm/rgw.rst @@ -82,6 +82,41 @@ something like: See :ref:`orchestrator-cli-placement-spec` for details of the placement specification. See :ref:`multisite` for more information of setting up multisite RGW. +Setting up HTTPS +---------------- + +In order to enable HTTPS for RGW services, apply a spec file following this scheme: + +.. code-block:: yaml + + service_type: rgw + service_id: myrgw + spec: + rgw_frontend_ssl_certificate: | + -----BEGIN PRIVATE KEY----- + V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFt + ZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15 + IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu + YSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3Mg + ZXQgYWNjdXNhbSBldCBqdXN0byBkdW8= + -----END PRIVATE KEY----- + -----BEGIN CERTIFICATE----- + V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFt + ZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15 + IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu + YSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3Mg + ZXQgYWNjdXNhbSBldCBqdXN0byBkdW8= + -----END CERTIFICATE----- + ssl: true + +Then apply this yaml document: + +.. prompt:: bash # + + ceph orch apply -i myrgw.yaml + +Note the value of ``rgw_frontend_ssl_certificate`` is a literal string as +indicated by a ``|`` character preserving newline characters. .. _orchestrator-haproxy-service-spec: diff --git a/ceph/doc/cephadm/service-management.rst b/ceph/doc/cephadm/service-management.rst index 02424d492..c8f47d9ed 100644 --- a/ceph/doc/cephadm/service-management.rst +++ b/ceph/doc/cephadm/service-management.rst @@ -158,6 +158,54 @@ or in a YAML files. cephadm will not deploy daemons on hosts with the ``_no_schedule`` label; see :ref:`cephadm-special-host-labels`. + .. note:: + The **apply** command can be confusing. For this reason, we recommend using + YAML specifications. + + Each ``ceph orch apply `` command supersedes the one before it. + If you do not use the proper syntax, you will clobber your work + as you go. + + For example: + + .. prompt:: bash # + + ceph orch apply mon host1 + ceph orch apply mon host2 + ceph orch apply mon host3 + + This results in only one host having a monitor applied to it: host 3. + + (The first command creates a monitor on host1. Then the second command + clobbers the monitor on host1 and creates a monitor on host2. Then the + third command clobbers the monitor on host2 and creates a monitor on + host3. In this scenario, at this point, there is a monitor ONLY on + host3.) + + To make certain that a monitor is applied to each of these three hosts, + run a command like this: + + .. prompt:: bash # + + ceph orch apply mon "host1,host2,host3" + + There is another way to apply monitors to multiple hosts: a ``yaml`` file + can be used. Instead of using the "ceph orch apply mon" commands, run a + command of this form: + + .. prompt:: bash # + + ceph orch apply -i file.yaml + + Here is a sample **file.yaml** file:: + + service_type: mon + placement: + hosts: + - host1 + - host2 + - host3 + Explicit placements ------------------- @@ -192,7 +240,39 @@ and ``=name`` specifies the name of the new monitor. Placement by labels ------------------- -Daemons can be explicitly placed on hosts that match a specific label: +Daemon placement can be limited to hosts that match a specific label. To set +a label ``mylabel`` to the appropriate hosts, run this command: + + .. prompt:: bash # + + ceph orch host label add ** mylabel + + To view the current hosts and labels, run this command: + + .. prompt:: bash # + + ceph orch host ls + + For example: + + .. prompt:: bash # + + ceph orch host label add host1 mylabel + ceph orch host label add host2 mylabel + ceph orch host label add host3 mylabel + ceph orch host ls + + .. code-block:: bash + + HOST ADDR LABELS STATUS + host1 mylabel + host2 mylabel + host3 mylabel + host4 + host5 + +Now, Tell cephadm to deploy daemons based on the label by running +this command: .. prompt:: bash # @@ -240,8 +320,8 @@ Or in YAML: host_pattern: "*" -Setting a limit ---------------- +Changing the number of monitors +------------------------------- By specifying ``count``, only the number of daemons specified will be created: @@ -402,7 +482,17 @@ To disable the automatic management of dameons, set ``unmanaged=True`` in the Deploying a daemon on a host manually ------------------------------------- -To manually deploy a daemon on a host, run a command of the following form: +.. note:: + + This workflow has a very limited use case and should only be used + in rare circumstances. + +To manually deploy a daemon on a host, follow these steps: + +Modify the service spec for a service by getting the +existing spec, adding ``unmanaged: true``, and applying the modified spec. + +Then manually deploy the daemon using the following: .. prompt:: bash # @@ -414,6 +504,13 @@ For example : ceph orch daemon add mgr --placement=my_host +.. note:: + + Removing ``unmanaged: true`` from the service spec will + enable the reconciliation loop for this service and will + potentially lead to the removal of the daemon, depending + on the placement spec. + Removing a daemon from a host manually -------------------------------------- diff --git a/ceph/doc/cephadm/troubleshooting.rst b/ceph/doc/cephadm/troubleshooting.rst index 5858d3940..1b6764dd7 100644 --- a/ceph/doc/cephadm/troubleshooting.rst +++ b/ceph/doc/cephadm/troubleshooting.rst @@ -1,46 +1,70 @@ Troubleshooting =============== -Sometimes there is a need to investigate why a cephadm command failed or why -a specific service no longer runs properly. +You might need to investigate why a cephadm command failed +or why a certain service no longer runs properly. -As cephadm deploys daemons as containers, troubleshooting daemons is slightly -different. Here are a few tools and commands to help investigating issues. +Cephadm deploys daemons as containers. This means that +troubleshooting those containerized daemons might work +differently than you expect (and that is certainly true if +you expect this troubleshooting to work the way that +troubleshooting does when the daemons involved aren't +containerized). + +Here are some tools and commands to help you troubleshoot +your Ceph environment. .. _cephadm-pause: Pausing or disabling cephadm ---------------------------- -If something goes wrong and cephadm is doing behaving in a way you do -not like, you can pause most background activity with:: +If something goes wrong and cephadm is behaving badly, you can +pause most of the Ceph cluster's background activity by running +the following command: + +.. prompt:: bash # ceph orch pause -This will stop any changes, but cephadm will still periodically check hosts to -refresh its inventory of daemons and devices. You can disable cephadm -completely with:: +This stops all changes in the Ceph cluster, but cephadm will +still periodically check hosts to refresh its inventory of +daemons and devices. You can disable cephadm completely by +running the following commands: + +.. prompt:: bash # ceph orch set backend '' ceph mgr module disable cephadm -This will disable all of the ``ceph orch ...`` CLI commands but the previously -deployed daemon containers will still continue to exist and start as they -did before. +These commands disable all of the ``ceph orch ...`` CLI commands. +All previously deployed daemon containers continue to exist and +will start as they did before you ran these commands. -Please refer to :ref:`cephadm-spec-unmanaged` for disabling individual -services. +See :ref:`cephadm-spec-unmanaged` for information on disabling +individual services. Per-service and per-daemon events --------------------------------- -In order to aid debugging failed daemon deployments, cephadm stores -events per service and per daemon. They often contain relevant information:: +In order to help with the process of debugging failed daemon +deployments, cephadm stores events per service and per daemon. +These events often contain information relevant to +troubleshooting +your Ceph cluster. + +Listing service events +~~~~~~~~~~~~~~~~~~~~~~ + +To see the events associated with a certain service, run a +command of the and following form: + +.. prompt:: bash # ceph orch ls --service_name= --format yaml -for example: +This will return something in the following form: .. code-block:: yaml @@ -58,10 +82,18 @@ for example: - '2021-02-01T12:09:25.264584 service:alertmanager [ERROR] "Failed to apply: Cannot place on unknown_host: Unknown hosts"' -Or per daemon:: +Listing daemon events +~~~~~~~~~~~~~~~~~~~~~ + +To see the events associated with a certain daemon, run a +command of the and following form: + +.. prompt:: bash # ceph orch ps --service-name --daemon-id --format yaml +This will return something in the following form: + .. code-block:: yaml daemon_type: mds @@ -77,16 +109,11 @@ Or per daemon:: Checking cephadm logs --------------------- -You can monitor the cephadm log in real time with:: - - ceph -W cephadm +To learn how to monitor the cephadm logs as they are generated, read :ref:`watching_cephadm_logs`. -You can see the last few messages with:: - - ceph log last cephadm - -If you have enabled logging to files, you can see a cephadm log file called -``ceph.cephadm.log`` on monitor hosts (see :ref:`cephadm-logs`). +If your Ceph cluster has been configured to log events to files, there will exist a +cephadm log file called ``ceph.cephadm.log`` on all monitor hosts (see +:ref:`cephadm-logs` for a more complete explanation of this). Gathering log files ------------------- @@ -190,7 +217,8 @@ Things users can do: [root@mon1 ~]# ssh -F config -i ~/cephadm_private_key root@mon1 Verifying that the Public Key is Listed in the authorized_keys file -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + To verify that the public key is in the authorized_keys file, run the following commands:: [root@mon1 ~]# cephadm shell -- ceph cephadm get-pub-key > ~/ceph.pub diff --git a/ceph/doc/cephadm/upgrade.rst b/ceph/doc/cephadm/upgrade.rst index 4e3e657bf..dd6bd8759 100644 --- a/ceph/doc/cephadm/upgrade.rst +++ b/ceph/doc/cephadm/upgrade.rst @@ -12,26 +12,32 @@ The automated upgrade process follows Ceph best practices. For example: * Each daemon is restarted only after Ceph indicates that the cluster will remain available. -Keep in mind that the Ceph cluster health status is likely to switch to -``HEALTH_WARNING`` during the upgrade. +.. note:: + + The Ceph cluster health status is likely to switch to + ``HEALTH_WARNING`` during the upgrade. + +.. note:: + + In case a host of the cluster is offline, the upgrade is paused. Starting the upgrade ==================== -Before you begin using cephadm to upgrade Ceph, verify that all hosts are currently online and that your cluster is healthy: +Before you use cephadm to upgrade Ceph, verify that all hosts are currently online and that your cluster is healthy by running the following command: .. prompt:: bash # ceph -s -To upgrade (or downgrade) to a specific release: +To upgrade (or downgrade) to a specific release, run the following command: .. prompt:: bash # ceph orch upgrade start --ceph-version -For example, to upgrade to v15.2.1: +For example, to upgrade to v15.2.1, run the following command: .. prompt:: bash # @@ -76,11 +82,11 @@ Watch the cephadm log by running the following command: Canceling an upgrade ==================== -You can stop the upgrade process at any time with: +You can stop the upgrade process at any time by running the following command: .. prompt:: bash # - # ceph orch upgrade stop + ceph orch upgrade stop Potential problems @@ -91,22 +97,27 @@ There are a few health alerts that can arise during the upgrade process. UPGRADE_NO_STANDBY_MGR ---------------------- -This alert means that Ceph requires an active and standby manager daemon in -order to proceed, but there is currently no standby. +This alert (``UPGRADE_NO_STANDBY_MGR``) means that Ceph does not detect an +active standby manager daemon. In order to proceed with the upgrade, Ceph +requires an active standby manager daemon (which you can think of in this +context as "a second manager"). -You can ensure that Cephadm is configured to run 2 (or more) managers by running the following command: +You can ensure that Cephadm is configured to run 2 (or more) managers by +running the following command: .. prompt:: bash # ceph orch apply mgr 2 # or more -You can check the status of existing mgr daemons by running the following command: +You can check the status of existing mgr daemons by running the following +command: .. prompt:: bash # ceph orch ps --daemon-type mgr -If an existing mgr daemon has stopped, you can try to restart it by running the following command: +If an existing mgr daemon has stopped, you can try to restart it by running the +following command: .. prompt:: bash # @@ -115,12 +126,13 @@ If an existing mgr daemon has stopped, you can try to restart it by running the UPGRADE_FAILED_PULL ------------------- -This alert means that Ceph was unable to pull the container image for the -target version. This can happen if you specify a version or container image -that does not exist (e.g. "1.2.3"), or if the container registry can not -be reached by one or more hosts in the cluster. +This alert (``UPGRADE_FAILED_PULL``) means that Ceph was unable to pull the +container image for the target version. This can happen if you specify a +version or container image that does not exist (e.g. "1.2.3"), or if the +container registry can not be reached by one or more hosts in the cluster. -To cancel the existing upgrade and to specify a different target version, run the following commands: +To cancel the existing upgrade and to specify a different target version, run +the following commands: .. prompt:: bash # diff --git a/ceph/doc/cephfs/administration.rst b/ceph/doc/cephfs/administration.rst index c53161313..fe55ff8ca 100644 --- a/ceph/doc/cephfs/administration.rst +++ b/ceph/doc/cephfs/administration.rst @@ -349,24 +349,6 @@ for use in exceptional circumstances. Incorrect use of these commands may cause serious problems, such as an inaccessible file system. -:: - - mds compat rm_compat - -Removes an compatibility feature flag. - -:: - - mds compat rm_incompat - -Removes an incompatibility feature flag. - -:: - - mds compat show - -Show MDS compatibility flags. - :: mds rmfailed @@ -379,3 +361,14 @@ This removes a rank from the failed set. This command resets the file system state to defaults, except for the name and pools. Non-zero ranks are saved in the stopped set. + + +:: + + fs new --fscid --force + +This command creates a file system with a specific **fscid** (file system cluster ID). +You may want to do this when an application expects the file system's ID to be +stable after it has been recovered, e.g., after monitor databases are lost and +rebuilt. Consequently, file system IDs don't always keep increasing with newer +file systems. diff --git a/ceph/doc/cephfs/fs-nfs-exports.rst b/ceph/doc/cephfs/fs-nfs-exports.rst index b11c88a19..1a95a1cb0 100644 --- a/ceph/doc/cephfs/fs-nfs-exports.rst +++ b/ceph/doc/cephfs/fs-nfs-exports.rst @@ -15,6 +15,53 @@ Requirements .. note:: From Pacific, the nfs mgr module must be enabled prior to use. +Ganesha Configuration Hierarchy +=============================== + +Cephadm and rook starts nfs-ganesha daemon with `bootstrap configuration` +containing minimal ganesha configuration, creates empty rados `common config` +object in `nfs-ganesha` pool and watches this config object. The `mgr/nfs` +module adds rados export object urls to the common config object. If cluster +config is set, it creates `user config` object containing custom ganesha +configuration and adds it url to common config object. + +.. ditaa:: + + + rados://$pool/$namespace/export-$i rados://$pool/$namespace/userconf-nfs.$cluster_id + (export config) (user config) + + +----------+ +----------+ +----------+ +---------------------------+ + | | | | | | | | + | export-1 | | export-2 | | export-3 | | userconf-nfs.$cluster_id | + | | | | | | | | + +----+-----+ +----+-----+ +-----+----+ +-------------+-------------+ + ^ ^ ^ ^ + | | | | + +--------------------------------+-------------------------+ + %url | + | + +--------+--------+ + | | rados://$pool/$namespace/conf-nfs.$svc + | conf+nfs.$svc | (common config) + | | + +--------+--------+ + ^ + | + watch_url | + +----------------------------------------------+ + | | | + | | | RADOS + +----------------------------------------------------------------------------------+ + | | | CONTAINER + watch_url | watch_url | watch_url | + | | | + +--------+-------+ +--------+-------+ +-------+--------+ + | | | | | | /etc/ganesha/ganesha.conf + | nfs.$svc.a | | nfs.$svc.b | | nfs.$svc.c | (bootstrap config) + | | | | | | + +----------------+ +----------------+ +----------------+ + Create NFS Ganesha Cluster ========================== diff --git a/ceph/doc/cephfs/upgrading.rst b/ceph/doc/cephfs/upgrading.rst index e9df8c0c7..2dc29e129 100644 --- a/ceph/doc/cephfs/upgrading.rst +++ b/ceph/doc/cephfs/upgrading.rst @@ -6,13 +6,11 @@ flags to support seamless upgrades of the MDSs without potentially causing assertions or other faults due to incompatible messages or other functional differences. For this reason, it's necessary during any cluster upgrade to reduce the number of active MDS for a file system to one first so that two -active MDS do not communicate with different versions. Further, it's also -necessary to take standbys offline as any new CompatSet flags will propagate -via the MDSMap to all MDS and cause older MDS to suicide. +active MDS do not communicate with different versions. The proper sequence for upgrading the MDS cluster is: -1. Disable and stop standby-replay daemons. +1. For each file system, disable and stop standby-replay daemons. :: @@ -27,7 +25,7 @@ command. Older versions of Ceph require you to stop these daemons manually. ceph mds fail mds. -2. Reduce the number of ranks to 1: +2. For each file system, reduce the number of ranks to 1: :: @@ -39,43 +37,20 @@ command. Older versions of Ceph require you to stop these daemons manually. ceph status # wait for MDS to finish stopping -4. Take all standbys offline, e.g. using systemctl: - -:: - - systemctl stop ceph-mds.target - -5. Confirm only one MDS is online and is rank 0 for your FS: - -:: - - ceph status - -6. Upgrade the single active MDS, e.g. using systemctl: - -:: - - # use package manager to update cluster - systemctl restart ceph-mds.target - -7. Upgrade/start the standby daemons. +4. For each MDS, upgrade packages and restart. Note: to reduce failovers, it is + recommended -- but not strictly necessary -- to first upgrade standby daemons. :: # use package manager to update cluster systemctl restart ceph-mds.target -8. Restore the previous max_mds for your cluster: +5. For each file system, restore the previous max_mds and allow_standby_replay settings for your cluster: :: ceph fs set max_mds - -9. Restore setting for ``allow_standby_replay`` (if applicable): - -:: - - ceph fs set allow_standby_replay true + ceph fs set allow_standby_replay Upgrading pre-Firefly file systems past Jewel diff --git a/ceph/doc/dev/cephadm/developing-cephadm.rst b/ceph/doc/dev/cephadm/developing-cephadm.rst index 9d6531d80..17e4d3dde 100644 --- a/ceph/doc/dev/cephadm/developing-cephadm.rst +++ b/ceph/doc/dev/cephadm/developing-cephadm.rst @@ -124,6 +124,20 @@ This means we should do very few synchronous calls to remote hosts. As a guideline, cephadm should do at most ``O(1)`` network calls in CLI handlers. Everything else should be done asynchronously in other threads, like ``serve()``. +Note regarding different variables used in the code +=================================================== + +* a ``service_type`` is something like mon, mgr, alertmanager etc defined + in ``ServiceSpec`` +* a ``service_id`` is the name of the service. Some services don't have + names. +* a ``service_name`` is ``.`` +* a ``daemon_type`` is the same as the service_type, except for ingress, + which has the haproxy and keepalived daemon types. +* a ``daemon_id`` is typically ``..``. + (Not the case for e.g. OSDs. OSDs are always called OSD.N) +* a ``daemon_name`` is ``.`` + Kcli: a virtualization management tool to make easy orchestrators development ============================================================================= `Kcli `_ is meant to interact with existing diff --git a/ceph/doc/dev/developer_guide/dash-devel.rst b/ceph/doc/dev/developer_guide/dash-devel.rst index 6a3e7976b..f892d29da 100644 --- a/ceph/doc/dev/developer_guide/dash-devel.rst +++ b/ceph/doc/dev/developer_guide/dash-devel.rst @@ -430,7 +430,14 @@ run-cephadm-e2e-tests.sh Orchestrator backend behave correctly. Prerequisites: you need to install `KCLI -`_ in your local machine. +`_ and Node.js in your local machine. + +Configure KCLI plan requirements:: + + $ sudo chown -R $(id -un) /var/lib/libvirt/images + $ mkdir -p /var/lib/libvirt/images/ceph-dashboard dashboard + $ kcli create pool -p /var/lib/libvirt/images/ceph-dashboard dashboard + $ kcli create network -c 192.168.100.0/24 dashboard Note: This script is aimed to be run as jenkins job so the cleanup is triggered only in a jenkins @@ -439,9 +446,26 @@ Note: Start E2E tests by running:: $ cd - $ sudo chown -R $(id -un) src/pybind/mgr/dashboard/frontend/dist src/pybind/mgr/dashboard/frontend/node_modules + $ sudo chown -R $(id -un) src/pybind/mgr/dashboard/frontend/{dist,node_modules,src/environments} + $ ./src/pybind/mgr/dashboard/ci/cephadm/run-cephadm-e2e-tests.sh + +You can also start a cluster in development mode (so the frontend build starts in watch mode and you +only have to reload the page for the changes to be reflected) by running:: + + $ ./src/pybind/mgr/dashboard/ci/cephadm/start-cluster.sh --dev-mode + +Note: + Add ``--expanded`` if you need a cluster ready to deploy services (one with enough monitor + daemons spread across different hosts and enough OSDs). + +Test your changes by running: + $ ./src/pybind/mgr/dashboard/ci/cephadm/run-cephadm-e2e-tests.sh - $ kcli delete plan -y ceph # After tests finish. + +Shutdown the cluster by running: + + $ kcli delete plan -y ceph + $ # In development mode, also kill the npm build watch process (e.g., pkill -f "ng build") Other running options ..................... @@ -1652,6 +1676,58 @@ load the controllers that we want to test. In the above example we are only loading the ``Ping`` controller. We can also disable authentication of a controller at this stage, as depicted in the example. +How to update or create new dashboards in grafana? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We are using ``jsonnet`` and ``grafonnet-lib`` to write code for the grafana dashboards. +All the dashboards are written inside ``grafana_dashboards.jsonnet`` file in the +monitoring/grafana/dashboards/jsonnet directory. + +We generate the dashboard json files directly from this jsonnet file by running this +command in the grafana/dashboards directory: +``jsonnet -m . jsonnet/grafana_dashboards.jsonnet``. +(For the above command to succeed we need ``jsonnet`` package installed and ``grafonnet-lib`` +directory cloned in our machine. Please refer - +``https://grafana.github.io/grafonnet-lib/getting-started/`` in case you have some trouble.) + +To update an existing grafana dashboard or to create a new one, we need to update +the ``grafana_dashboards.jsonnet`` file and generate the new/updated json files using the +above mentioned command. For people who are not familiar with grafonnet or jsonnet implementation +can follow this doc - ``https://grafana.github.io/grafonnet-lib/``. + +Example grafana dashboard in jsonnet format: + +To specify the grafana dashboard properties such as title, uid etc we can create a local function - + +:: + + local dashboardSchema(title, uid, time_from, refresh, schemaVersion, tags,timezone, timepicker) + +To add a graph panel we can spcify the graph schema in a local function such as - + +:: + + local graphPanelSchema(title, nullPointMode, stack, formatY1, formatY2, labelY1, labelY2, min, fill, datasource) + +and then use these functions inside the dashboard definition like - + +:: + + { + radosgw-sync-overview.json: //json file name to be generated + + dashboardSchema( + 'RGW Sync Overview', 'rgw-sync-overview', 'now-1h', '15s', .., .., .. + ) + + .addPanels([ + graphPanelSchema( + 'Replication (throughput) from Source Zone', 'Bps', null, .., .., ..) + ]) + } + +The valid grafonnet-lib attributes can be found here - ``https://grafana.github.io/grafonnet-lib/api-docs/``. + How to listen for manager notifications in a controller? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/ceph/doc/install/containers.rst b/ceph/doc/install/containers.rst index 2c4efd00e..936be0be0 100644 --- a/ceph/doc/install/containers.rst +++ b/ceph/doc/install/containers.rst @@ -19,11 +19,11 @@ Ceph Container Images Official Releases ----------------- -Ceph Container images are available from Docker Hub at:: +Ceph Container images are available from both Quay and Docker Hub:: + https://quay.io/repository/ceph/ceph https://hub.docker.com/r/ceph - ceph/ceph ^^^^^^^^^ @@ -42,6 +42,13 @@ ceph/ceph | vRELNUM.Y.Z-YYYYMMDD | A specific build (e.g., *v14.2.4-20191203*) | +----------------------+--------------------------------------------------------------+ +Legacy container images +----------------------- + +Legacy container images are available from Docker Hub at:: + + https://hub.docker.com/r/ceph + ceph/daemon-base ^^^^^^^^^^^^^^^^ diff --git a/ceph/doc/man/8/ceph-volume.rst b/ceph/doc/man/8/ceph-volume.rst index b3c70a556..5a28695ae 100644 --- a/ceph/doc/man/8/ceph-volume.rst +++ b/ceph/doc/man/8/ceph-volume.rst @@ -15,7 +15,7 @@ Synopsis | **ceph-volume** **inventory** | **ceph-volume** **lvm** [ *trigger* | *create* | *activate* | *prepare* -| *zap* | *list* | *batch*] +| *zap* | *list* | *batch* | *new-wal* | *new-db* | *migrate* ] | **ceph-volume** **simple** [ *trigger* | *scan* | *activate* ] @@ -241,6 +241,96 @@ Positional arguments: ``/path/to/sda1`` or ``/path/to/sda`` for regular devices. +new-wal +^^^^^^^ + +Attaches the given logical volume to OSD as a WAL. Logical volume +name format is vg/lv. Fails if OSD has already got attached WAL. + +Usage:: + + ceph-volume lvm new-wal --osd-id OSD_ID --osd-fsid OSD_FSID --target + +Optional arguments: + +.. option:: -h, --help + + show the help message and exit + +.. option:: --no-systemd + + Skip checking OSD systemd unit + +Required arguments: + +.. option:: --target + + logical volume name to attach as WAL + +new-db +^^^^^^ + +Attaches the given logical volume to OSD as a DB. Logical volume +name format is vg/lv. Fails if OSD has already got attached DB. + +Usage:: + + ceph-volume lvm new-db --osd-id OSD_ID --osd-fsid OSD_FSID --target + +Optional arguments: + +.. option:: -h, --help + + show the help message and exit + +.. option:: --no-systemd + + Skip checking OSD systemd unit + +Required arguments: + +.. option:: --target + + logical volume name to attach as DB + +migrate +^^^^^^^ + +Moves BlueFS data from source volume(s) to the target one, source volumes +(except the main, i.e. data or block one) are removed on success. LVM volumes +are permitted for Target only, both already attached or new one. In the latter +case it is attached to the OSD replacing one of the source devices. Following +replacement rules apply (in the order of precedence, stop on the first match): + + - if source list has DB volume - target device replaces it. + - if source list has WAL volume - target device replace it. + - if source list has slow volume only - operation is not permitted, + requires explicit allocation via new-db/new-wal command. + +Usage:: + + ceph-volume lvm migrate --osd-id OSD_ID --osd-fsid OSD_FSID --target --from {data|db|wal} [{data|db|wal} ...] + +Optional arguments: + +.. option:: -h, --help + + show the help message and exit + +.. option:: --no-systemd + + Skip checking OSD systemd unit + +Required arguments: + +.. option:: --from + + list of source device type names + +.. option:: --target + + logical volume to move data to + simple ------ diff --git a/ceph/doc/man/8/cephadm.rst b/ceph/doc/man/8/cephadm.rst index db0c0698c..ebcc3a6a2 100644 --- a/ceph/doc/man/8/cephadm.rst +++ b/ceph/doc/man/8/cephadm.rst @@ -53,6 +53,7 @@ Synopsis | **cephadm** **bootstrap** [-h] [--config CONFIG] [--mon-id MON_ID] | [--mon-addrv MON_ADDRV] [--mon-ip MON_IP] | [--mgr-id MGR_ID] [--fsid FSID] +| [--log-to-file] [--single-host-defaults] | [--output-dir OUTPUT_DIR] | [--output-keyring OUTPUT_KEYRING] | [--output-config OUTPUT_CONFIG] @@ -126,13 +127,14 @@ Options .. option:: --docker use docker instead of podman (default: False) -.. option::data-dir DATA_DIR - base directory for daemon data (default:/var/lib/ceph) +.. option:: --data-dir DATA_DIR + + base directory for daemon data (default: /var/lib/ceph) .. option:: --log-dir LOG_DIR - base directory for daemon logs (default:.. option:: /var/log/ceph) + base directory for daemon logs (default: /var/log/ceph) .. option:: --logrotate-dir LOGROTATE_DIR @@ -208,6 +210,8 @@ Arguments: * [--mon-ip MON_IP] mon IP * [--mgr-id MGR_ID] mgr id (default: randomly generated) * [--fsid FSID] cluster FSID +* [--log-to-file] configure cluster to log to traditional log files +* [--single-host-defaults] configure cluster to run on a single host * [--output-dir OUTPUT_DIR] directory to write config, keyring, and pub key files * [--output-keyring OUTPUT_KEYRING] location to write keyring file with new cluster admin and mon keys * [--output-config OUTPUT_CONFIG] location to write conf file to connect to new cluster diff --git a/ceph/doc/mgr/dashboard.rst b/ceph/doc/mgr/dashboard.rst index 32d02fd4a..3ac0e0333 100644 --- a/ceph/doc/mgr/dashboard.rst +++ b/ceph/doc/mgr/dashboard.rst @@ -376,50 +376,17 @@ password. Enabling the Object Gateway Management Frontend ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -To use the Object Gateway management functionality of the dashboard, you will -need to provide the login credentials of a user with the ``system`` flag -enabled. If you do not have a ``system`` user already, you must create one:: +When RGW is deployed with cephadm, the RGW credentials used by the +dashboard will be automatically configured. You can also manually force the +credentials to be set up with:: - $ radosgw-admin user create --uid= --display-name= \ - --system + $ ceph dashboard set-rgw-credentials -Take note of the keys ``access_key`` and ``secret_key`` in the output. +This will create an RGW user with uid ``dashboard`` for each realm in +the system. -To obtain the credentials of an existing user via `radosgw-admin`:: +If you've configured a custom 'admin' resource in your RGW admin API, you should set it here also:: - $ radosgw-admin user info --uid= - -In case of having several Object Gateways, you will need the required users' credentials -to connect to each Object Gateway. -Finally, provide these credentials to the dashboard:: - - $ echo -n "{'': '', '': '', ...}" > - $ echo -n "{'': '', '': '', ...}" > - $ ceph dashboard set-rgw-api-access-key -i - $ ceph dashboard set-rgw-api-secret-key -i - -.. note:: - - Legacy way of providing credentials (connect to single Object Gateway):: - - $ echo -n "" > - $ echo -n "" > - -In a simple configuration with a single RGW endpoint, this is all you -have to do to get the Object Gateway management functionality working. The -dashboard will try to automatically determine the host and port -from the Ceph Manager's service map. - -In case of having several Object Gateways, you might want to set -the default one by setting its host and port manually:: - - $ ceph dashboard set-rgw-api-host - $ ceph dashboard set-rgw-api-port - -In addition to the settings mentioned so far, the following settings do also -exist and you may find yourself in the situation that you have to use them:: - - $ ceph dashboard set-rgw-api-scheme # http or https $ ceph dashboard set-rgw-api-admin-resource If you are using a self-signed certificate in your Object Gateway setup, @@ -1314,6 +1281,7 @@ and loosely coupled fashion. .. include:: dashboard_plugins/feature_toggles.inc.rst .. include:: dashboard_plugins/debug.inc.rst +.. include:: dashboard_plugins/motd.inc.rst Troubleshooting the Dashboard diff --git a/ceph/doc/mgr/dashboard_plugins/motd.inc.rst b/ceph/doc/mgr/dashboard_plugins/motd.inc.rst new file mode 100644 index 000000000..b8464e1f3 --- /dev/null +++ b/ceph/doc/mgr/dashboard_plugins/motd.inc.rst @@ -0,0 +1,30 @@ +.. _dashboard-motd: + +Message of the day (MOTD) +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Displays a configured `message of the day` at the top of the Ceph Dashboard. + +The importance of a MOTD can be configured by its severity, which is +`info`, `warning` or `danger`. The MOTD can expire after a given time, +this means it will not be displayed in the UI anymore. Use the following +syntax to specify the expiration time: `Ns|m|h|d|w` for seconds, minutes, +hours, days and weeks. If the MOTD should expire after 2 hours, use `2h` +or `5w` for 5 weeks. Use `0` to configure a MOTD that does not expire. + +To configure a MOTD, run the following command:: + + $ ceph dashboard motd set + +To show the configured MOTD:: + + $ ceph dashboard motd get + +To clear the configured MOTD run:: + + $ ceph dashboard motd clear + +A MOTD with a `info` or `warning` severity can be closed by the user. The +`info` MOTD is not displayed anymore until the local storage cookies are +cleared or a new MOTD with a different severity is displayed. A MOTD with +a 'warning' severity will be displayed again in a new session. diff --git a/ceph/doc/rados/operations/balancer.rst b/ceph/doc/rados/operations/balancer.rst index 5146c23c2..76ae9830d 100644 --- a/ceph/doc/rados/operations/balancer.rst +++ b/ceph/doc/rados/operations/balancer.rst @@ -40,9 +40,37 @@ healed itself). When the cluster is healthy, the balancer will throttle its changes such that the percentage of PGs that are misplaced (i.e., that need to be moved) is below a threshold of (by default) 5%. The -``max_misplaced`` threshold can be adjusted with:: +``target_max_misplaced_ratio`` threshold can be adjusted with:: - ceph config set mgr mgr/balancer/max_misplaced .07 # 7% + ceph config set mgr target_max_misplaced_ratio .07 # 7% + +Set the number of seconds to sleep in between runs of the automatic balancer:: + + ceph config set mgr mgr/balancer/sleep_interval 60 + +Set the time of day to begin automatic balancing in HHMM format:: + + ceph config set mgr mgr/balancer/begin_time 0000 + +Set the time of day to finish automatic balancing in HHMM format:: + + ceph config set mgr mgr/balancer/end_time 2400 + +Restrict automatic balancing to this day of the week or later. +Uses the same conventions as crontab, 0 or 7 is Sunday, 1 is Monday, and so on:: + + ceph config set mgr mgr/balancer/begin_weekday 0 + +Restrict automatic balancing to this day of the week or earlier. +Uses the same conventions as crontab, 0 or 7 is Sunday, 1 is Monday, and so on:: + + ceph config set mgr mgr/balancer/end_weekday 7 + +Pool IDs to which the automatic balancing will be limited. +The default for this is an empty string, meaning all pools will be balanced. +The numeric pool IDs can be gotten with the :command:`ceph osd pool ls detail` command:: + + ceph config set mgr mgr/balancer/pool_ids 1,2,3 Modes @@ -136,3 +164,4 @@ The quality of the distribution that would result after executing a plan can be Assuming the plan is expected to improve the distribution (i.e., it has a lower score than the current cluster state), the user can execute that plan with:: ceph balancer execute + diff --git a/ceph/doc/rados/operations/monitoring.rst b/ceph/doc/rados/operations/monitoring.rst index d38dc088d..d154ca2b9 100644 --- a/ceph/doc/rados/operations/monitoring.rst +++ b/ceph/doc/rados/operations/monitoring.rst @@ -410,10 +410,9 @@ on the number of replicas, clones and snapshots. to this pool. - **QUOTA OBJECTS:** The number of quota objects. - **QUOTA BYTES:** The number of bytes in the quota objects. -- **DIRTY:** "DIRTY" is meaningful only when cache tiering is in use. If cache - tiering is in use, the "DIRTY" column lists the number of objects in the - cache pool that have been written to the cache pool but have not flushed yet - to the base pool. +- **DIRTY:** The number of objects in the cache pool that have been written to + the cache pool but have not been flushed yet to the base pool. This field is + only available when cache tiering is in use. - **USED COMPR:** amount of space allocated for compressed data (i.e. this includes comrpessed data plus all the allocation, replication and erasure coding overhead). diff --git a/ceph/doc/rados/operations/placement-groups.rst b/ceph/doc/rados/operations/placement-groups.rst index 699c26f8e..947fdb156 100644 --- a/ceph/doc/rados/operations/placement-groups.rst +++ b/ceph/doc/rados/operations/placement-groups.rst @@ -41,10 +41,10 @@ the PG count with this command:: Output will be something like:: - POOL SIZE TARGET SIZE RATE RAW CAPACITY RATIO TARGET RATIO EFFECTIVE RATIO PG_NUM NEW PG_NUM AUTOSCALE - a 12900M 3.0 82431M 0.4695 8 128 warn - c 0 3.0 82431M 0.0000 0.2000 0.9884 1 64 warn - b 0 953.6M 3.0 82431M 0.0347 8 warn + POOL SIZE TARGET SIZE RATE RAW CAPACITY RATIO TARGET RATIO EFFECTIVE RATIO BIAS PG_NUM NEW PG_NUM AUTOSCALE PROFILE + a 12900M 3.0 82431M 0.4695 8 128 warn scale-up + c 0 3.0 82431M 0.0000 0.2000 0.9884 1.0 1 64 warn scale-down + b 0 953.6M 3.0 82431M 0.0347 8 warn scale-down **SIZE** is the amount of data stored in the pool. **TARGET SIZE**, if present, is the amount of data the administrator has specified that @@ -77,6 +77,10 @@ ratio takes precedence. The system uses the larger of the actual ratio and the effective ratio for its calculation. +**BIAS** is used as a multiplier to manually adjust a pool's PG based +on prior information about how much PGs a specific pool is expected +to have. + **PG_NUM** is the current number of PGs for the pool (or the current number of PGs that the pool is working towards, if a ``pg_num`` change is in progress). **NEW PG_NUM**, if present, is what the @@ -84,9 +88,13 @@ system believes the pool's ``pg_num`` should be changed to. It is always a power of 2, and will only be present if the "ideal" value varies from the current value by more than a factor of 3. -The final column, **AUTOSCALE**, is the pool ``pg_autoscale_mode``, +**AUTOSCALE**, is the pool ``pg_autoscale_mode`` and will be either ``on``, ``off``, or ``warn``. +The final column, **PROFILE** shows the autoscale profile +used by each pool. ``scale-up`` and ``scale-down`` are the +currently available profiles. + Automated scaling ----------------- @@ -113,6 +121,28 @@ example, a pool that maps to OSDs of class `ssd` and a pool that maps to OSDs of class `hdd` will each have optimal PG counts that depend on the number of those respective device types. +The autoscaler uses the `scale-down` profile by default, +where each pool starts out with a full complements of PGs and only scales +down when the usage ratio across the pools is not even. However, it also has +a `scale-up` profile, where it starts out each pool with minimal PGs and scales +up PGs when there is more usage in each pool. + +With only the `scale-down` profile, the autoscaler identifies +any overlapping roots and prevents the pools with such roots +from scaling because overlapping roots can cause problems +with the scaling process. + +To use the `scale-up` profile:: + + ceph osd pool set autoscale-profile scale-up + +To switch back to the default `scale-down` profile:: + + ceph osd pool set autoscale-profile scale-down + +Existing clusters will continue to use the `scale-up` profile. +To use the `scale-down` profile, users will need to set autoscale-profile `scale-down`, +after upgrading to a version of Ceph that provides the `scale-down` feature. .. _specifying_pool_target_size: diff --git a/ceph/doc/radosgw/frontends.rst b/ceph/doc/radosgw/frontends.rst index e4a013590..274cdce87 100644 --- a/ceph/doc/radosgw/frontends.rst +++ b/ceph/doc/radosgw/frontends.rst @@ -64,6 +64,38 @@ Options :Type: String :Default: None +``ssl_options`` + +:Description: Optional colon separated list of ssl context options: + + ``default_workarounds`` Implement various bug workarounds. + + ``no_compression`` Disable compression. + + ``no_sslv2`` Disable SSL v2. + + ``no_sslv3`` Disable SSL v3. + + ``no_tlsv1`` Disable TLS v1. + + ``no_tlsv1_1`` Disable TLS v1.1. + + ``no_tlsv1_2`` Disable TLS v1.2. + + ``single_dh_use`` Always create a new key when using tmp_dh parameters. + +:Type: String +:Default: ``no_sslv2:no_sslv3:no_tlsv1:no_tlsv1_1`` + +``ssl_ciphers`` + +:Description: Optional list of one or more cipher strings separated by colons. + The format of the string is described in openssl's ciphers(1) + manual. + +:Type: String +:Default: None + ``tcp_nodelay`` :Description: If set the socket option will disable Nagle's algorithm on @@ -100,6 +132,7 @@ Civetweb ======== .. versionadded:: Firefly +.. deprecated:: Pacific The ``civetweb`` frontend uses the Civetweb HTTP library, which is a fork of Mongoose. diff --git a/ceph/doc/radosgw/vault.rst b/ceph/doc/radosgw/vault.rst index 840bc5a09..0f3cb8fd1 100644 --- a/ceph/doc/radosgw/vault.rst +++ b/ceph/doc/radosgw/vault.rst @@ -400,6 +400,19 @@ Or, when using the transit secret engine:: In the example above, the Gateway would only fetch transit encryption keys under ``https://vault-server:8200/v1/transit``. +You can use custom ssl certs to authenticate with vault with help of +following options:: + + rgw crypt vault verify ssl = true + rgw crypt vault ssl cacert = /etc/ceph/vault.ca + rgw crypt vault ssl clientcert = /etc/ceph/vault.crt + rgw crypt vault ssl clientkey = /etc/ceph/vault.key + +where vault.ca is CA certificate and vault.key/vault.crt are private key and ssl +ceritificate generated for RGW to access the vault server. It highly recommended to +set this option true, setting false is very dangerous and need to avoid since this +runs in very secured enviroments. + Transit engine compatibility support ------------------------------------ The transit engine has compatibility support for previous diff --git a/ceph/make-debs.sh b/ceph/make-debs.sh index 5a7ca1aff..ed79df777 100755 --- a/ceph/make-debs.sh +++ b/ceph/make-debs.sh @@ -16,9 +16,9 @@ # set -xe +. /etc/os-release base=${1:-/tmp/release} -codename=$(lsb_release -sc) -releasedir=$base/$(lsb_release -si)/WORKDIR +releasedir=$base/$NAME/WORKDIR rm -fr $(dirname $releasedir) mkdir -p $releasedir # @@ -60,7 +60,7 @@ dvers="$vers-1" cd ceph-$vers chvers=$(head -1 debian/changelog | perl -ne 's/.*\(//; s/\).*//; print') if [ "$chvers" != "$dvers" ]; then - DEBEMAIL="contact@ceph.com" dch -D $codename --force-distribution -b -v "$dvers" "new version" + DEBEMAIL="contact@ceph.com" dch -D $VERSION_CODENAME --force-distribution -b -v "$dvers" "new version" fi # # create the packages @@ -74,18 +74,18 @@ if test $NPROC -gt 1 ; then fi PATH=/usr/lib/ccache:$PATH dpkg-buildpackage $j -uc -us cd ../.. -mkdir -p $codename/conf -cat > $codename/conf/distributions < $VERSION_CODENAME/conf/distributions < $codename/version +echo $dvers > $VERSION_CODENAME/version diff --git a/ceph/make-dist b/ceph/make-dist index 2e97a1a12..ff6a4a27e 100755 --- a/ceph/make-dist +++ b/ceph/make-dist @@ -163,7 +163,7 @@ tar cvf $outfile.version.tar $outfile/src/.git_version $outfile/ceph.spec # at the three URLs referenced below (may involve uploading to download.ceph.com) boost_version=1.73.0 download_boost $boost_version 4eb3b8d442b426dc35346235c8733b5ae35ba431690e38c6a8263dce9fcbb402 \ - https://dl.bintray.com/boostorg/release/$boost_version/source \ + https://boostorg.jfrog.io/artifactory/main/release/$boost_version/source \ https://downloads.sourceforge.net/project/boost/boost/$boost_version \ https://download.ceph.com/qa download_liburing 0.7 8e2842cfe947f3a443af301bdd6d034455536c38a455c7a700d0c1ad165a7543 \ diff --git a/ceph/monitoring/grafana/build/Makefile b/ceph/monitoring/grafana/build/Makefile index fd96a827f..c0cda1130 100755 --- a/ceph/monitoring/grafana/build/Makefile +++ b/ceph/monitoring/grafana/build/Makefile @@ -1,33 +1,38 @@ -GRAFANA_VERSION := 6.7.4-1 -PIECHART_VERSION := "1.4.0" -STATUS_PANEL_VERSION := "1.0.9" -DASHBOARD_DIR := "monitoring/grafana/dashboards" +GRAFANA_VERSION ?= 6.7.4-1 +PIECHART_VERSION ?= "1.4.0" +STATUS_PANEL_VERSION ?= "1.0.9" +DASHBOARD_DIR := "../dashboards" DASHBOARD_PROVISIONING := "ceph-dashboard.yml" -IMAGE := "centos:8" -VERSION := "${IMAGE: -1}" +IMAGE := "docker.io/centos:8" PKGMGR := "dnf" -# CONTAINER := $(shell buildah from ${IMAGE}) GF_CONFIG := "/etc/grafana/grafana.ini" -ceph_version := "master" +# clip off "- from the end of GRAFANA_VERSION +CONTAINER_VERSION := $(shell /bin/echo $(GRAFANA_VERSION) | /bin/sed 's/-.*//') + +ARCH ?= x86_64 +ifeq "$(ARCH)" "arm64" + override ARCH := aarch64 +endif + +LOCALTAG=ceph-grafana:$(CONTAINER_VERSION)-$(ARCH) +TAG=ceph/ceph-grafana:$(CONTAINER_VERSION)-$(ARCH) # Build a grafana instance - preconfigured for use within Ceph's dashboard UI -build : fetch_dashboards +build : echo "Creating base container" - $(eval CONTAINER := $(shell buildah from ${IMAGE})) + $(eval CONTAINER := $(shell sudo buildah from ${IMAGE})) # Using upstream grafana build - wget https://dl.grafana.com/oss/release/grafana-${GRAFANA_VERSION}.x86_64.rpm - #wget localhost:8000/grafana-${GRAFANA_VERSION}.x86_64.rpm - #cp grafana-${GRAFANA_VERSION}.x86_64.rpm ${mountpoint}/tmp/. - buildah copy $(CONTAINER) grafana-${GRAFANA_VERSION}.x86_64.rpm /tmp/grafana-${GRAFANA_VERSION}.x86_64.rpm - buildah run $(CONTAINER) ${PKGMGR} install -y --setopt install_weak_deps=false --setopt=tsflags=nodocs /tmp/grafana-${GRAFANA_VERSION}.x86_64.rpm - buildah run $(CONTAINER) ${PKGMGR} clean all - buildah run $(CONTAINER) rm -f /tmp/grafana*.rpm - buildah run $(CONTAINER) grafana-cli plugins install grafana-piechart-panel ${PIECHART_VERSION} - buildah run $(CONTAINER) grafana-cli plugins install vonage-status-panel ${STATUS_PANEL_VERSION} - buildah run $(CONTAINER) mkdir -p /etc/grafana/dashboards/ceph-dashboard - buildah copy $(CONTAINER) jsonfiles/*.json /etc/grafana/dashboards/ceph-dashboard + curl -fLO https://dl.grafana.com/oss/release/grafana-${GRAFANA_VERSION}.${ARCH}.rpm + sudo buildah copy $(CONTAINER) grafana-${GRAFANA_VERSION}.${ARCH}.rpm /tmp/grafana-${GRAFANA_VERSION}.${ARCH}.rpm + sudo buildah run $(CONTAINER) ${PKGMGR} install -y --setopt install_weak_deps=false --setopt=tsflags=nodocs /tmp/grafana-${GRAFANA_VERSION}.${ARCH}.rpm + sudo buildah run $(CONTAINER) ${PKGMGR} clean all + sudo buildah run $(CONTAINER) rm -f /tmp/grafana*.rpm + sudo buildah run $(CONTAINER) grafana-cli plugins install grafana-piechart-panel ${PIECHART_VERSION} + sudo buildah run $(CONTAINER) grafana-cli plugins install vonage-status-panel ${STATUS_PANEL_VERSION} + sudo buildah run $(CONTAINER) mkdir -p /etc/grafana/dashboards/ceph-dashboard + sudo buildah copy $(CONTAINER) ${DASHBOARD_DIR}/*.json /etc/grafana/dashboards/ceph-dashboard @/bin/echo -e "\ apiVersion: 1 \\n\ @@ -43,55 +48,49 @@ providers: \\n\ path: '/etc/grafana/dashboards/ceph-dashboard'" >> ${DASHBOARD_PROVISIONING} - buildah copy $(CONTAINER) ${DASHBOARD_PROVISIONING} /etc/grafana/provisioning/dashboards/${DASHBOARD_PROVISIONING} + sudo buildah copy $(CONTAINER) ${DASHBOARD_PROVISIONING} /etc/grafana/provisioning/dashboards/${DASHBOARD_PROVISIONING} # expose tcp/3000 for grafana - buildah config --port 3000 $(CONTAINER) + sudo buildah config --port 3000 $(CONTAINER) # set working dir - buildah config --workingdir /usr/share/grafana $(CONTAINER) + sudo buildah config --workingdir /usr/share/grafana $(CONTAINER) # set environment overrides from the default locations in /usr/share - buildah config --env GF_PATHS_LOGS="/var/log/grafana" $(CONTAINER) - buildah config --env GF_PATHS_PLUGINS="/var/lib/grafana/plugins" $(CONTAINER) - buildah config --env GF_PATHS_PROVISIONING="/etc/grafana/provisioning" $(CONTAINER) - buildah config --env GF_PATHS_DATA="/var/lib/grafana" $(CONTAINER) + sudo buildah config --env GF_PATHS_LOGS="/var/log/grafana" $(CONTAINER) + sudo buildah config --env GF_PATHS_PLUGINS="/var/lib/grafana/plugins" $(CONTAINER) + sudo buildah config --env GF_PATHS_PROVISIONING="/etc/grafana/provisioning" $(CONTAINER) + sudo buildah config --env GF_PATHS_DATA="/var/lib/grafana" $(CONTAINER) # entrypoint - buildah config --entrypoint "grafana-server --config=${GF_CONFIG}" $(CONTAINER) + sudo buildah config --entrypoint "grafana-server --config=${GF_CONFIG}" $(CONTAINER) # finalize - buildah config --label maintainer="Paul Cuzner " $(CONTAINER) - buildah config --label description="Ceph Grafana Container" $(CONTAINER) - buildah config --label summary="Grafana Container configured for Ceph mgr/dashboard integration" $(CONTAINER) - buildah commit --format docker --squash $(CONTAINER) ceph-grafana:${ceph_version} - buildah tag ceph-grafana:${ceph_version} ceph/ceph-grafana:${ceph_version} - - -fetch_dashboards: clean - wget -O - https://api.github.com/repos/ceph/ceph/contents/${DASHBOARD_DIR}?ref=${ceph_version} | jq '.[].download_url' > dashboards - - # drop quotes from the list and pick out only json files - sed -i 's/\"//g' dashboards - sed -i '/\.json/!d' dashboards - mkdir jsonfiles - while read -r line; do \ - wget "$$line" -P jsonfiles; \ - done < dashboards - -clean : - rm -f dashboards - rm -fr jsonfiles - rm -f grafana-*.rpm* + sudo buildah config --label maintainer="Paul Cuzner " $(CONTAINER) + sudo buildah config --label description="Ceph Grafana Container" $(CONTAINER) + sudo buildah config --label summary="Grafana Container configured for Ceph mgr/dashboard integration" $(CONTAINER) + sudo buildah commit --format docker --squash $(CONTAINER) $(LOCALTAG) + +push: + # this transition-through-oci image is a workaround for + # https://github.com/containers/buildah/issues/3253 and + # can be removed when that is fixed and released. The + # --format v2s2 on push is to convert oci back to docker format. + sudo podman push $(LOCALTAG) --format=oci dir://tmp/oci-image + sudo podman pull dir://tmp/oci-image + sudo rm -rf /tmp/oci-image + sudo podman tag localhost/tmp/oci-image docker.io/${TAG} + sudo podman tag localhost/tmp/oci-image quay.io/${TAG} + # sudo podman has issues with auth.json; just override it + sudo podman login --authfile=auth.json -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} docker.io + sudo podman login --authfile=auth.json -u $(CONTAINER_REPO_USERNAME) -p $(CONTAINER_REPO_PASSWORD) quay.io + sudo podman push --authfile=auth.json --format v2s2 docker.io/${TAG} + sudo podman push --authfile=auth.json --format v2s2 quay.io/${TAG} + +clean: + sudo podman rmi ${LOCALTAG} || true + sudo podman rmi docker.io/${TAG} || true + sudo podman rmi quay.io/${TAG} || true + sudo podman rmi localhost/tmp/oci-image || true + rm -f grafana-*.rpm* auth.json rm -f ${DASHBOARD_PROVISIONING} - - -nautilus : - $(MAKE) ceph_version="nautilus" build -octopus : - $(MAKE) ceph_version="octopus" build -master : - $(MAKE) ceph_version="master" build - -all : nautilus octopus master -.PHONY : all diff --git a/ceph/monitoring/grafana/dashboards/CMakeLists.txt b/ceph/monitoring/grafana/dashboards/CMakeLists.txt index 41ca947a6..65037cbd8 100644 --- a/ceph/monitoring/grafana/dashboards/CMakeLists.txt +++ b/ceph/monitoring/grafana/dashboards/CMakeLists.txt @@ -1,8 +1,34 @@ set(CEPH_GRAFANA_DASHBOARDS_DIR "${CMAKE_INSTALL_SYSCONFDIR}/grafana/dashboards/ceph-dashboard" CACHE PATH "Location for grafana dashboards") - -FILE(GLOB CEPH_GRAFANA_DASHBOARDS "*.json") - +file(GLOB CEPH_GRAFANA_DASHBOARDS "*.json") install(FILES ${CEPH_GRAFANA_DASHBOARDS} DESTINATION ${CEPH_GRAFANA_DASHBOARDS_DIR}) + +set(CEPH_BUILD_VIRTUALENV $ENV{TMPDIR}) +if(NOT CEPH_BUILD_VIRTUALENV) + set(CEPH_BUILD_VIRTUALENV ${CMAKE_BINARY_DIR}) +endif() + +if(WITH_GRAFANA) + include(AddCephTest) + add_tox_test(grafana TOX_ENVS grafonnet-check) + set(ver 0.1.0) + set(name grafonnet-lib) + include(ExternalProject) + ExternalProject_Add(${name} + URL https://github.com/grafana/${name}/archive/v${ver}/${name}-${ver}.tar.gz + URL_MD5 0798752ed40864fa8b3db40a3c970642 + BUILD_COMMAND "" + CONFIGURE_COMMAND "" + INSTALL_COMMAND "" + EXCLUDE_FROM_ALL ON) + add_dependencies(tests + ${name}) + ExternalProject_Get_Property(${name} SOURCE_DIR) + set_property( + TEST run-tox-grafana + APPEND + PROPERTY ENVIRONMENT + GRAFONNET_PATH=${SOURCE_DIR}/grafonnet) +endif() diff --git a/ceph/monitoring/grafana/dashboards/ceph-cluster.json b/ceph/monitoring/grafana/dashboards/ceph-cluster.json index 447dbb293..e4e367efd 100644 --- a/ceph/monitoring/grafana/dashboards/ceph-cluster.json +++ b/ceph/monitoring/grafana/dashboards/ceph-cluster.json @@ -107,8 +107,9 @@ "tableColumn": "", "targets": [ { - "expr": "ceph_health_status{instance=~'$instance'}", + "expr": "ceph_health_status", "format": "time_series", + "instant": true, "interval": "$interval", "intervalFactor": 1, "refId": "A", @@ -174,7 +175,7 @@ "displayAliasType": "Always", "displayType": "Regular", "displayValueWithAlias": "When Alias Displayed", - "expr": "count(ceph_osd_metadata{instance=~\"$instance\"})", + "expr": "count(ceph_osd_metadata)", "format": "time_series", "intervalFactor": 1, "legendFormat": "All", @@ -189,7 +190,7 @@ "displayAliasType": "Always", "displayType": "Regular", "displayValueWithAlias": "When Alias Displayed", - "expr": "sum(ceph_osds_in{instance=~\"$instance\"})", + "expr": "sum(ceph_osds_in)", "format": "time_series", "intervalFactor": 1, "legendFormat": "In", @@ -204,7 +205,7 @@ "displayAliasType": "Warning / Critical", "displayType": "Regular", "displayValueWithAlias": "When Alias Displayed", - "expr": "sum(ceph_osd_in{instance=~\"$instance\"} == bool 0)", + "expr": "sum(ceph_osd_in == bool 0)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -221,7 +222,7 @@ "displayAliasType": "Always", "displayType": "Regular", "displayValueWithAlias": "When Alias Displayed", - "expr": "sum(ceph_osd_up{instance=~\"$instance\"})", + "expr": "sum(ceph_osd_up)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Up", @@ -237,7 +238,7 @@ "displayAliasType": "Warning / Critical", "displayType": "Regular", "displayValueWithAlias": "When Alias Displayed", - "expr": "sum(ceph_osd_up{instance=~\"$instance\"} == bool 0)", + "expr": "sum(ceph_osd_up == bool 0)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Down", @@ -263,7 +264,7 @@ "decimals": 2, "format": "percentunit", "gauge": { - "maxValue": 100, + "maxValue": 1, "minValue": 0, "show": true, "thresholdLabels": false, @@ -312,14 +313,14 @@ "tableColumn": "", "targets": [ { - "expr": "sum(ceph_osd_stat_bytes_used{instance=~\"$instance\"})/sum(ceph_osd_stat_bytes{instance=~\"$instance\"})", + "expr": "sum(ceph_osd_stat_bytes_used)/sum(ceph_osd_stat_bytes)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Used", "refId": "A" } ], - "thresholds": "70,80", + "thresholds": "0.7,0.8", "title": "Capacity used", "type": "singlestat", "valueFontSize": "80%", @@ -530,28 +531,28 @@ "steppedLine": false, "targets": [ { - "expr": "quantile(0.95, ceph_osd_apply_latency_ms{instance=~\"$instance\"})", + "expr": "quantile(0.95, ceph_osd_apply_latency_ms)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Apply Latency P_95", "refId": "A" }, { - "expr": "quantile(0.95, ceph_osd_commit_latency_ms{instance=~\"$instance\"})", + "expr": "quantile(0.95, ceph_osd_commit_latency_ms)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Commit Latency P_95", "refId": "B" }, { - "expr": "avg(ceph_osd_apply_latency_ms{instance=~\"$instance\"})", + "expr": "avg(ceph_osd_apply_latency_ms)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Avg Apply Latency", "refId": "C" }, { - "expr": "avg(ceph_osd_commit_latency_ms{instance=~\"$instance\"})", + "expr": "avg(ceph_osd_commit_latency_ms)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Avg Commit Latency", @@ -629,7 +630,7 @@ "displayAliasType": "Always", "displayType": "Regular", "displayValueWithAlias": "When Alias Displayed", - "expr": "sum(ceph_mon_quorum_status{instance=~\"$instance\"})", + "expr": "sum(ceph_mon_quorum_status)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -646,7 +647,7 @@ "displayAliasType": "Always", "displayType": "Regular", "displayValueWithAlias": "When Alias Displayed", - "expr": "count(ceph_mon_quorum_status{instance=~\"$instance\"})", + "expr": "count(ceph_mon_quorum_status)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Total", @@ -663,7 +664,7 @@ "displayAliasType": "Warning / Critical", "displayType": "Annotation", "displayValueWithAlias": "Never", - "expr": "count(ceph_mon_quorum_status{instance=~\"$instance\"}) / sum(ceph_mon_quorum_status{instance=~\"$instance\"})", + "expr": "count(ceph_mon_quorum_status) / sum(ceph_mon_quorum_status)", "format": "time_series", "intervalFactor": 1, "legendFormat": "MONs out of Quorum", @@ -710,7 +711,7 @@ "displayAliasType": "Always", "displayType": "Regular", "displayValueWithAlias": "When Alias Displayed", - "expr": "ceph_mds_server_handle_client_session{instance=~\"$instance\"}", + "expr": "ceph_mds_server_handle_client_session", "format": "time_series", "intervalFactor": 1, "legendFormat": "Clients", @@ -764,14 +765,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(ceph_osd_op_w_in_bytes{instance=~\"$instance\"}[1m]))", + "expr": "sum(irate(ceph_osd_op_w_in_bytes[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Writes", "refId": "A" }, { - "expr": "sum(irate(ceph_osd_op_r_out_bytes{instance=~\"$instance\"}[1m]))", + "expr": "sum(irate(ceph_osd_op_r_out_bytes[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Reads", @@ -851,7 +852,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(deriv(ceph_pool_stored{instance=~\"$instance\"}[1m]))", + "expr": "sum(deriv(ceph_pool_stored[1m]))", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -924,7 +925,7 @@ "span": 12, "targets": [ { - "expr": "ceph_osd_stat_bytes_used{instance=~'$instance'} / ceph_osd_stat_bytes{instance=~'$instance'}", + "expr": "ceph_osd_stat_bytes_used / ceph_osd_stat_bytes", "format": "time_series", "interval": "1m", "intervalFactor": 1, @@ -946,7 +947,7 @@ "xBucketNumber": null, "xBucketSize": "", "yAxis": { - "decimals": null, + "decimals": 2, "format": "percentunit", "logBase": 1, "max": null, @@ -986,7 +987,7 @@ "links": [], "targets": [ { - "expr": "ceph_osd_numpg{instance=~\"$instance\"}", + "expr": "ceph_osd_numpg", "format": "time_series", "intervalFactor": 1, "legendFormat": "#PGs", @@ -1190,29 +1191,6 @@ "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", "refresh": 2, "type": "interval" - }, - { - "allFormat": "glob", - "allValue": null, - "current": {}, - "datasource": "$datasource", - "hide": 0, - "hideLabel": false, - "includeAll": true, - "label": "Exporter Instance", - "multi": false, - "multiFormat": "glob", - "name": "instance", - "options": [], - "query": "label_values(ceph_health_status, instance)", - "refresh": 1, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false } ] }, @@ -1245,7 +1223,7 @@ "30d" ] }, - "timezone": "browser", + "timezone": "", "title": "Ceph - Cluster", "version": 13 } diff --git a/ceph/monitoring/grafana/dashboards/host-details.json b/ceph/monitoring/grafana/dashboards/host-details.json index 237349dd3..71ac36f37 100644 --- a/ceph/monitoring/grafana/dashboards/host-details.json +++ b/ceph/monitoring/grafana/dashboards/host-details.json @@ -1208,7 +1208,7 @@ "30d" ] }, - "timezone": "browser", + "timezone": "", "title": "Host Details", "uid": "rtOg0AiWz", "version": 4 diff --git a/ceph/monitoring/grafana/dashboards/jsonnet/grafana_dashboards.jsonnet b/ceph/monitoring/grafana/dashboards/jsonnet/grafana_dashboards.jsonnet new file mode 100644 index 000000000..11c89b212 --- /dev/null +++ b/ceph/monitoring/grafana/dashboards/jsonnet/grafana_dashboards.jsonnet @@ -0,0 +1,54 @@ +local g = import 'grafana.libsonnet'; + +local dashboardSchema(title, uid, time_from, refresh, schemaVersion, tags,timezone, timepicker) = + g.dashboard.new(title=title, uid=uid, time_from=time_from, refresh=refresh, schemaVersion=schemaVersion, tags=tags, timezone=timezone, timepicker=timepicker); + +local graphPanelSchema(title, nullPointMode, stack, formatY1, formatY2, labelY1, labelY2, min, fill, datasource) = + g.graphPanel.new(title=title, nullPointMode=nullPointMode, stack=stack, formatY1=formatY1, formatY2=formatY2, labelY1=labelY1, labelY2=labelY2, min=min, fill=fill, datasource=datasource); + +local addTargetSchema(expr, intervalFactor, format, legendFormat) = + g.prometheus.target(expr=expr, intervalFactor=intervalFactor, format=format, legendFormat=legendFormat); + +local addTemplateSchema(name, datasource, query, refresh, hide, includeAll, sort) = + g.template.new(name=name, datasource=datasource, query=query, refresh=refresh, hide=hide, includeAll=includeAll, sort=sort); + +local addAnnotationSchema(builtIn, datasource, enable, hide, iconColor, name, type) = + g.annotation.datasource(builtIn=builtIn, datasource=datasource, enable=enable, hide=hide, iconColor=iconColor, name=name, type=type); + +{ + "radosgw-sync-overview.json": + local RgwSyncOverviewPanel(title, formatY1, labelY1, rgwMetric, x, y, w, h) = + graphPanelSchema(title, 'null as zero', true, formatY1, 'short', labelY1, null, 0, 1, '$datasource') + .addTargets( + [addTargetSchema('sum by (source_zone) (rate(%s[30s]))' % rgwMetric, 1, 'time_series', '{{source_zone}}')]) + {gridPos: {x: x, y: y, w: w, h: h}}; + + dashboardSchema( + 'RGW Sync Overview', 'rgw-sync-overview', 'now-1h', '15s', 16, ["overview"], '', {refresh_intervals:['5s','10s','15s','30s','1m','5m','15m','30m','1h','2h','1d'],time_options:['5m','15m','1h','6h','12h','24h','2d','7d','30d']} + ) + .addAnnotation( + addAnnotationSchema( + 1, '-- Grafana --', true, true, 'rgba(0, 211, 255, 1)', 'Annotations & Alerts', 'dashboard') + ) + .addRequired( + type='grafana', id='grafana', name='Grafana', version='5.0.0' + ) + .addRequired( + type='panel', id='graph', name='Graph', version='5.0.0' + ) + .addTemplate( + addTemplateSchema('rgw_servers', '$datasource', 'prometehus', 1, 2, true, 1) + ) + .addTemplate( + g.template.datasource('datasource', 'prometheus', 'default', label='Data Source') + ) + .addPanels([ + RgwSyncOverviewPanel( + 'Replication (throughput) from Source Zone', 'Bps', null, 'ceph_data_sync_from_zone_fetch_bytes_sum', 0, 0, 8, 7), + RgwSyncOverviewPanel( + 'Replication (objects) from Source Zone', 'short', 'Objects/s', 'ceph_data_sync_from_zone_fetch_bytes_count', 8, 0, 8, 7), + RgwSyncOverviewPanel( + 'Polling Request Latency from Source Zone', 'ms', null, 'ceph_data_sync_from_zone_poll_latency_sum', 16, 0, 8, 7), + RgwSyncOverviewPanel( + 'Unsuccessful Object Replications from Source Zone', 'short', 'Count/s', 'ceph_data_sync_from_zone_fetch_errors', 0, 7, 8, 7) + ]) +} diff --git a/ceph/monitoring/grafana/dashboards/osd-device-details.json b/ceph/monitoring/grafana/dashboards/osd-device-details.json index deb6ed7d2..eefb59125 100644 --- a/ceph/monitoring/grafana/dashboards/osd-device-details.json +++ b/ceph/monitoring/grafana/dashboards/osd-device-details.json @@ -423,7 +423,7 @@ }, "yaxes": [ { - "format": "ms", + "format": "s", "label": "Read (-) / Write (+)", "logBase": 1, "max": null, diff --git a/ceph/monitoring/grafana/dashboards/pool-detail.json b/ceph/monitoring/grafana/dashboards/pool-detail.json index 41554ed30..dd6bc3927 100644 --- a/ceph/monitoring/grafana/dashboards/pool-detail.json +++ b/ceph/monitoring/grafana/dashboards/pool-detail.json @@ -658,7 +658,7 @@ "30d" ] }, - "timezone": "browser", + "timezone": "", "title": "Ceph Pool Details", "uid": "-xyV8KCiz", "version": 1 diff --git a/ceph/monitoring/grafana/dashboards/pool-overview.json b/ceph/monitoring/grafana/dashboards/pool-overview.json index cd699348b..c405f6075 100644 --- a/ceph/monitoring/grafana/dashboards/pool-overview.json +++ b/ceph/monitoring/grafana/dashboards/pool-overview.json @@ -1554,7 +1554,7 @@ "30d" ] }, - "timezone": "browser", + "timezone": "", "title": "Ceph Pools Overview", "uid": "z99hzWtmk", "variables": { diff --git a/ceph/monitoring/grafana/dashboards/radosgw-sync-overview.json b/ceph/monitoring/grafana/dashboards/radosgw-sync-overview.json index e9136d78e..d6cd5e98a 100644 --- a/ceph/monitoring/grafana/dashboards/radosgw-sync-overview.json +++ b/ceph/monitoring/grafana/dashboards/radosgw-sync-overview.json @@ -1,440 +1,455 @@ { - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ + "__inputs": [ ], + "__requires": [ { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "id": null, - "iteration": 1534386107523, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (source_zone) (rate(ceph_data_sync_from_zone_fetch_bytes_sum[30s]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{source_zone}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Replication (throughput) from Source Zone", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "unit": "bytes", - "format": "Bps", - "decimals": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - "h": 7, - "w": 7.4, - "x": 8.3, - "y": 0 - }, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (source_zone) (rate(ceph_data_sync_from_zone_fetch_bytes_count[30s]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{source_zone}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Replication (objects) from Source Zone", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "decimals": null, - "label": "Objects/s", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "5.0.0" }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (source_zone) (rate(ceph_data_sync_from_zone_poll_latency_sum[30s]) * 1000)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{source_zone}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Polling Request Latency from Source Zone", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "unit": "s", - "format": "ms", - "decimals": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } + { + "id": "graph", + "name": "Graph", + "type": "panel", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "showIn": 0, + "tags": [ ], + "type": "dashboard" + } ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 7 - }, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (source_zone) (rate(ceph_data_sync_from_zone_fetch_errors[30s]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{source_zone}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Unsuccessful Object Replications from Source Zone", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ ], + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (source_zone) (rate(ceph_data_sync_from_zone_fetch_bytes_sum[30s]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{source_zone}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Replication (throughput) from Source Zone", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (source_zone) (rate(ceph_data_sync_from_zone_fetch_bytes_count[30s]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{source_zone}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Replication (objects) from Source Zone", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": "Objects/s", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] }, - "yaxes": [ - { - "format": "short", - "decimals": null, - "label": "Count/s", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "refresh": "15s", - "schemaVersion": 16, - "style": "dark", - "tags": [ - "overview" - ], - "templating": { - "list": [ { - "allValue": null, - "current": {}, - "datasource": "$datasource", - "hide": 2, - "includeAll": true, - "label": null, - "multi": false, - "name": "rgw_servers", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (source_zone) (rate(ceph_data_sync_from_zone_poll_latency_sum[30s]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{source_zone}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Polling Request Latency from Source Zone", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] }, { - "current": { - "tags": [], - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Data Source", - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (source_zone) (rate(ceph_data_sync_from_zone_fetch_errors[30s]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{source_zone}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Unsuccessful Object Replications from Source Zone", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": "Count/s", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "15s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "RGW Sync Overview", - "uid": "rgw-sync-overview", - "version": 2 + ], + "refresh": "15s", + "rows": [ ], + "schemaVersion": 16, + "style": "dark", + "tags": [ + "overview" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 2, + "includeAll": true, + "label": null, + "multi": false, + "name": "rgw_servers", + "options": [ ], + "query": "prometehus", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "15s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "RGW Sync Overview", + "uid": "rgw-sync-overview", + "version": 0 } diff --git a/ceph/monitoring/grafana/dashboards/rbd-details.json b/ceph/monitoring/grafana/dashboards/rbd-details.json index 68fffad1e..59932a5ee 100644 --- a/ceph/monitoring/grafana/dashboards/rbd-details.json +++ b/ceph/monitoring/grafana/dashboards/rbd-details.json @@ -1,409 +1,409 @@ -{ - "__inputs": [], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.3.3" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Detailed Performance of RBD Images (IOPS/Throughput/Latency)", - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "id": null, - "iteration": 1584428820779, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$Datasource", - "fill": 1, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 6, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(ceph_rbd_write_ops{pool=\"$Pool\", image=\"$Image\"}[30s])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Write", - "refId": "A" - }, - { - "expr": "irate(ceph_rbd_read_ops{pool=\"$Pool\", image=\"$Image\"}[30s])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Read", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "IOPS", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "iops", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "iops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": true, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$Datasource", - "fill": 1, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(ceph_rbd_write_bytes{pool=\"$Pool\", image=\"$Image\"}[30s])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Write", - "refId": "A" - }, - { - "expr": "irate(ceph_rbd_read_bytes{pool=\"$Pool\", image=\"$Image\"}[30s])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Read", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Throughput", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": true, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$Datasource", - "fill": 1, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(ceph_rbd_write_latency_sum{pool=\"$Pool\", image=\"$Image\"}[30s]) / irate(ceph_rbd_write_latency_count{pool=\"$Pool\", image=\"$Image\"}[30s])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Write", - "refId": "A" - }, - { - "expr": "irate(ceph_rbd_read_latency_sum{pool=\"$Pool\", image=\"$Image\"}[30s]) / irate(ceph_rbd_read_latency_count{pool=\"$Pool\", image=\"$Image\"}[30s])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Read", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Average Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ns", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "ns", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": true, - "alignLevel": null - } - } - ], - "refresh": false, - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": {}, - "hide": 0, - "label": null, - "name": "Datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": null, - "current": {}, - "datasource": "$Datasource", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "Pool", - "options": [], - "query": "label_values(pool)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": "$Datasource", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "Image", - "options": [], - "query": "label_values(image)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "RBD Details", - "uid": "YhCYGcuZz", - "version": 7 -} \ No newline at end of file +{ + "__inputs": [], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.3.3" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Detailed Performance of RBD Images (IOPS/Throughput/Latency)", + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1584428820779, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$Datasource", + "fill": 1, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(ceph_rbd_write_ops{pool=\"$Pool\", image=\"$Image\"}[30s])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "irate(ceph_rbd_read_ops{pool=\"$Pool\", image=\"$Image\"}[30s])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "IOPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "iops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": true, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$Datasource", + "fill": 1, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(ceph_rbd_write_bytes{pool=\"$Pool\", image=\"$Image\"}[30s])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "irate(ceph_rbd_read_bytes{pool=\"$Pool\", image=\"$Image\"}[30s])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Throughput", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": true, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$Datasource", + "fill": 1, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(ceph_rbd_write_latency_sum{pool=\"$Pool\", image=\"$Image\"}[30s]) / irate(ceph_rbd_write_latency_count{pool=\"$Pool\", image=\"$Image\"}[30s])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "irate(ceph_rbd_read_latency_sum{pool=\"$Pool\", image=\"$Image\"}[30s]) / irate(ceph_rbd_read_latency_count{pool=\"$Pool\", image=\"$Image\"}[30s])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "ns", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": true, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "label": null, + "name": "Datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "$Datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "Pool", + "options": [], + "query": "label_values(pool)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "$Datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "Image", + "options": [], + "query": "label_values(image)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "RBD Details", + "uid": "YhCYGcuZz", + "version": 7 +} diff --git a/ceph/monitoring/grafana/dashboards/requirements-grafonnet.txt b/ceph/monitoring/grafana/dashboards/requirements-grafonnet.txt new file mode 100644 index 000000000..9891d5590 --- /dev/null +++ b/ceph/monitoring/grafana/dashboards/requirements-grafonnet.txt @@ -0,0 +1 @@ +jsondiff diff --git a/ceph/monitoring/grafana/dashboards/test-jsonnet.sh b/ceph/monitoring/grafana/dashboards/test-jsonnet.sh new file mode 100644 index 000000000..127992c05 --- /dev/null +++ b/ceph/monitoring/grafana/dashboards/test-jsonnet.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -e +TEMPDIR=`mktemp -d` +BASEDIR=$(dirname "$0") + +JSONNET_PATH="${GRAFONNET_PATH}" jsonnet -m ${TEMPDIR} $BASEDIR/jsonnet/grafana_dashboards.jsonnet + +truncate -s 0 ${TEMPDIR}/json_difference.log +for json_files in $BASEDIR/*.json +do + JSON_FILE_NAME=$(basename $json_files) + for generated_files in ${TEMPDIR}/*.json + do + GENERATED_FILE_NAME=$(basename $generated_files) + if [ $JSON_FILE_NAME == $GENERATED_FILE_NAME ]; then + jsondiff --indent 2 $generated_files $json_files | tee -a ${TEMPDIR}/json_difference.log + fi + done +done + +if [[ $(wc -l < ${TEMPDIR}/json_difference.log) -eq 0 ]] +then + rm -rf ${TEMPDIR} + echo "Congratulations! Grafonnet Check Passed" +else + rm -rf ${TEMPDIR} + echo "Grafonnet Check Failed, failed comparing generated file with existing" + exit 1 +fi diff --git a/ceph/monitoring/grafana/dashboards/tox.ini b/ceph/monitoring/grafana/dashboards/tox.ini new file mode 100644 index 000000000..10aeb9f38 --- /dev/null +++ b/ceph/monitoring/grafana/dashboards/tox.ini @@ -0,0 +1,22 @@ +[tox] +envlist = grafonnet-{check,fix} +skipsdist = true + +[grafonnet] +deps = + -rrequirements-grafonnet.txt + +[testenv:grafonnet-{check,fix}] +basepython = python3 +whitelist_externals = + jsonnet + bash +description = + check: Ensure that auto-generated grafana dashboard files matches the current version + fix: generate dashboard json files from jsonnet file with latest changes +deps = + {[grafonnet]deps} +passenv = GRAFONNET_PATH +commands = + check: bash test-jsonnet.sh + fix: jsonnet -m . jsonnet/grafana_dashboards.jsonnet diff --git a/ceph/qa/distros/podman/centos_8.2_container_tools_3.0.yaml b/ceph/qa/distros/podman/centos_8.2_container_tools_3.0.yaml new file mode 100644 index 000000000..6d27d157b --- /dev/null +++ b/ceph/qa/distros/podman/centos_8.2_container_tools_3.0.yaml @@ -0,0 +1,14 @@ +os_type: centos +os_version: "8.2" +overrides: + selinux: + whitelist: + - scontext=system_u:system_r:logrotate_t:s0 + +tasks: +- pexec: + all: + - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup + - sudo dnf -y module reset container-tools + - sudo dnf -y module install container-tools:3.0 + - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf diff --git a/ceph/qa/distros/podman/centos_8.2_kubic_stable.yaml b/ceph/qa/distros/podman/centos_8.2_kubic_stable.yaml deleted file mode 100644 index 22fbc1997..000000000 --- a/ceph/qa/distros/podman/centos_8.2_kubic_stable.yaml +++ /dev/null @@ -1,18 +0,0 @@ -os_type: centos -os_version: "8.2" -overrides: - selinux: - whitelist: - - scontext=system_u:system_r:logrotate_t:s0 - -tasks: -- pexec: - all: - - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - - sudo dnf -y module disable container-tools - - sudo dnf -y install 'dnf-command(copr)' - - sudo dnf -y copr enable rhcontainerbot/container-selinux - - sudo curl -L -o /etc/yum.repos.d/devel:kubic:libcontainers:stable.repo https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_8/devel:kubic:libcontainers:stable.repo - - sudo dnf remove -y podman - - sudo dnf -y install podman - - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf diff --git a/ceph/qa/distros/podman/centos_8.3_container_tools_3.0.yaml b/ceph/qa/distros/podman/centos_8.3_container_tools_3.0.yaml new file mode 100644 index 000000000..e2cad5e20 --- /dev/null +++ b/ceph/qa/distros/podman/centos_8.3_container_tools_3.0.yaml @@ -0,0 +1,14 @@ +os_type: centos +os_version: "8.3" +overrides: + selinux: + whitelist: + - scontext=system_u:system_r:logrotate_t:s0 + +tasks: +- pexec: + all: + - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup + - sudo dnf -y module reset container-tools + - sudo dnf -y module install container-tools:3.0 + - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf diff --git a/ceph/qa/suites/rgw/ignore-pg-availability.yaml b/ceph/qa/rgw/ignore-pg-availability.yaml similarity index 100% rename from ceph/qa/suites/rgw/ignore-pg-availability.yaml rename to ceph/qa/rgw/ignore-pg-availability.yaml diff --git a/ceph/qa/standalone/osd/osd-bluefs-volume-ops.sh b/ceph/qa/standalone/osd/osd-bluefs-volume-ops.sh index 86da6af5c..3e92d17db 100755 --- a/ceph/qa/standalone/osd/osd-bluefs-volume-ops.sh +++ b/ceph/qa/standalone/osd/osd-bluefs-volume-ops.sh @@ -8,16 +8,6 @@ function run() { local dir=$1 shift - export CEPH_MON="127.0.0.1:7146" # git grep '\<7146\>' : there must be only one - export CEPH_ARGS - CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--mon-host=$CEPH_MON " - CEPH_ARGS+="--bluestore_block_size=2147483648 " - CEPH_ARGS+="--bluestore_block_db_create=true " - CEPH_ARGS+="--bluestore_block_db_size=1073741824 " - CEPH_ARGS+="--bluestore_block_wal_size=536870912 " - CEPH_ARGS+="--bluestore_block_wal_create=true " - CEPH_ARGS+="--bluestore_fsck_on_mount=true " local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} for func in $funcs ; do setup $dir || return 1 @@ -33,6 +23,16 @@ function TEST_bluestore() { if [ $flimit -lt 1536 ]; then echo "Low open file limit ($flimit), test may fail. Increase to 1536 or higher and retry if that happens." fi + export CEPH_MON="127.0.0.1:7146" # git grep '\<7146\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + CEPH_ARGS+="--bluestore_block_size=2147483648 " + CEPH_ARGS+="--bluestore_block_db_create=true " + CEPH_ARGS+="--bluestore_block_db_size=1073741824 " + CEPH_ARGS+="--bluestore_block_wal_size=536870912 " + CEPH_ARGS+="--bluestore_block_wal_create=true " + CEPH_ARGS+="--bluestore_fsck_on_mount=true " run_mon $dir a || return 1 run_mgr $dir x || return 1 @@ -337,6 +337,63 @@ function TEST_bluestore() { wait_for_clean || return 1 } +function TEST_bluestore2() { + local dir=$1 + + local flimit=$(ulimit -n) + if [ $flimit -lt 1536 ]; then + echo "Low open file limit ($flimit), test may fail. Increase to 1536 or higher and retry if that happens." + fi + export CEPH_MON="127.0.0.1:7146" # git grep '\<7146\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + CEPH_ARGS+="--bluestore_block_size=4294967296 " + CEPH_ARGS+="--bluestore_block_db_create=true " + CEPH_ARGS+="--bluestore_block_db_size=1073741824 " + CEPH_ARGS+="--bluestore_block_wal_create=false " + CEPH_ARGS+="--bluestore_fsck_on_mount=true " + CEPH_ARGS+="--osd_pool_default_size=1 " + CEPH_ARGS+="--osd_pool_default_min_size=1 " + CEPH_ARGS+="--bluestore_debug_enforce_settings=ssd " + + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + osd_pid0=$(cat $dir/osd.0.pid) + + sleep 5 + create_pool foo 16 + + # write some objects + timeout 60 rados bench -p foo 10 write --write-omap --no-cleanup #|| return 1 + + #give RocksDB some time to cooldown and put files to slow level(s) + sleep 10 + + spilled_over=$( ceph tell osd.0 perf dump bluefs | jq ".bluefs.slow_used_bytes" ) + test $spilled_over -gt 0 || return 1 + + while kill $osd_pid0; do sleep 1 ; done + ceph osd down 0 + + ceph-bluestore-tool --path $dir/0 \ + --devs-source $dir/0/block.db \ + --dev-target $dir/0/block \ + --command bluefs-bdev-migrate || return 1 + + ceph-bluestore-tool --path $dir/0 \ + --command bluefs-bdev-sizes || return 1 + + ceph-bluestore-tool --path $dir/0 \ + --command fsck || return 1 + + activate_osd $dir 0 || return 1 + osd_pid0=$(cat $dir/osd.0.pid) + + wait_for_clean || return 1 +} + main osd-bluefs-volume-ops "$@" # Local Variables: diff --git a/ceph/qa/standalone/osd/osd-force-create-pg.sh b/ceph/qa/standalone/osd/osd-force-create-pg.sh index 855f479ff..ca4b0239e 100755 --- a/ceph/qa/standalone/osd/osd-force-create-pg.sh +++ b/ceph/qa/standalone/osd/osd-force-create-pg.sh @@ -12,14 +12,15 @@ function run() { local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} for func in $funcs ; do + setup $dir || return 1 $func $dir || return 1 + teardown $dir || return 1 done } function TEST_reuse_id() { local dir=$1 - setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 --mon_allow_pool_size_one=true || return 1 run_mgr $dir x || return 1 run_osd $dir 0 || return 1 diff --git a/ceph/qa/standalone/osd/osd-reuse-id.sh b/ceph/qa/standalone/osd/osd-reuse-id.sh index 92aa055a8..b24b6f2eb 100755 --- a/ceph/qa/standalone/osd/osd-reuse-id.sh +++ b/ceph/qa/standalone/osd/osd-reuse-id.sh @@ -27,14 +27,15 @@ function run() { local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} for func in $funcs ; do + setup $dir || return 1 $func $dir || return 1 + teardown $dir || return 1 done } function TEST_reuse_id() { local dir=$1 - setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 --mon_allow_pool_size_one=true || return 1 run_mgr $dir x || return 1 run_osd $dir 0 || return 1 diff --git a/ceph/qa/standalone/osd/pg-split-merge.sh b/ceph/qa/standalone/osd/pg-split-merge.sh index a8517a41b..7f2899b60 100755 --- a/ceph/qa/standalone/osd/pg-split-merge.sh +++ b/ceph/qa/standalone/osd/pg-split-merge.sh @@ -12,14 +12,15 @@ function run() { local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} for func in $funcs ; do + setup $dir || return 1 $func $dir || return 1 + teardown $dir || return 1 done } function TEST_a_merge_empty() { local dir=$1 - setup $dir || return 1 run_mon $dir a --osd_pool_default_size=3 || return 1 run_mgr $dir x || return 1 run_osd $dir 0 || return 1 @@ -87,7 +88,6 @@ function TEST_a_merge_empty() { function TEST_import_after_merge_and_gap() { local dir=$1 - setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 --mon_allow_pool_size_one=true || return 1 run_mgr $dir x || return 1 run_osd $dir 0 || return 1 @@ -162,7 +162,6 @@ function TEST_import_after_merge_and_gap() { function TEST_import_after_split() { local dir=$1 - setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 --mon_allow_pool_size_one=true || return 1 run_mgr $dir x || return 1 run_osd $dir 0 || return 1 diff --git a/ceph/qa/standalone/scrub/osd-scrub-repair.sh b/ceph/qa/standalone/scrub/osd-scrub-repair.sh index 1461d0d6c..e000134a8 100755 --- a/ceph/qa/standalone/scrub/osd-scrub-repair.sh +++ b/ceph/qa/standalone/scrub/osd-scrub-repair.sh @@ -60,7 +60,9 @@ function run() { export -n CEPH_CLI_TEST_DUP_COMMAND local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} for func in $funcs ; do + setup $dir || return 1 $func $dir || return 1 + teardown $dir || return 1 done } @@ -91,7 +93,6 @@ function TEST_corrupt_and_repair_replicated() { local dir=$1 local poolname=rbd - setup $dir || return 1 run_mon $dir a --osd_pool_default_size=2 || return 1 run_mgr $dir x || return 1 run_osd $dir 0 || return 1 @@ -103,8 +104,6 @@ function TEST_corrupt_and_repair_replicated() { corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1 # Reproduces http://tracker.ceph.com/issues/8914 corrupt_and_repair_one $dir $poolname $(get_primary $poolname SOMETHING) || return 1 - - teardown $dir || return 1 } # @@ -114,7 +113,6 @@ function TEST_allow_repair_during_recovery() { local dir=$1 local poolname=rbd - setup $dir || return 1 run_mon $dir a --osd_pool_default_size=2 || return 1 run_mgr $dir x || return 1 run_osd $dir 0 --osd_scrub_during_recovery=false \ @@ -128,8 +126,6 @@ function TEST_allow_repair_during_recovery() { add_something $dir $poolname || return 1 corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1 - - teardown $dir || return 1 } # @@ -139,7 +135,6 @@ function TEST_skip_non_repair_during_recovery() { local dir=$1 local poolname=rbd - setup $dir || return 1 run_mon $dir a --osd_pool_default_size=2 || return 1 run_mgr $dir x || return 1 run_osd $dir 0 --osd_scrub_during_recovery=false \ @@ -153,8 +148,6 @@ function TEST_skip_non_repair_during_recovery() { add_something $dir $poolname || return 1 scrub_and_not_schedule $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1 - - teardown $dir || return 1 } function scrub_and_not_schedule() { @@ -276,7 +269,6 @@ function auto_repair_erasure_coded() { local poolname=ecpool # Launch a cluster with 5 seconds scrub interval - setup $dir || return 1 run_mon $dir a || return 1 run_mgr $dir x || return 1 local ceph_osd_args="--osd-scrub-auto-repair=true \ @@ -285,11 +277,11 @@ function auto_repair_erasure_coded() { --osd-scrub-min-interval=5 \ --osd-scrub-interval-randomize-ratio=0" for id in $(seq 0 2) ; do - if [ "$allow_overwrites" = "true" ]; then + if [ "$allow_overwrites" = "true" ]; then run_osd $dir $id $ceph_osd_args || return 1 - else + else run_osd_filestore $dir $id $ceph_osd_args || return 1 - fi + fi done create_rbd_pool || return 1 wait_for_clean || return 1 @@ -314,9 +306,6 @@ function auto_repair_erasure_coded() { objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1 rados --pool $poolname get SOMETHING $dir/COPY || return 1 diff $dir/ORIGINAL $dir/COPY || return 1 - - # Tear down - teardown $dir || return 1 } function TEST_auto_repair_erasure_coded_appends() { @@ -329,16 +318,135 @@ function TEST_auto_repair_erasure_coded_overwrites() { fi } +# initiate a scrub, then check for the (expected) 'scrubbing' and the +# (not expected until an error was identified) 'repair' +# Arguments: osd#, pg, sleep time +function initiate_and_fetch_state() { + local the_osd="osd.$1" + local pgid=$2 + local last_scrub=$(get_last_scrub_stamp $pgid) + + set_config "osd" "$1" "osd_scrub_sleep" "$3" + set_config "osd" "$1" "osd_scrub_auto_repair" "true" + + flush_pg_stats + date --rfc-3339=ns + + # note: must initiate a "regular" (periodic) deep scrub - not an operator-initiated one + env CEPH_ARGS= ceph --format json daemon $(get_asok_path $the_osd) deep_scrub "$pgid" + env CEPH_ARGS= ceph --format json daemon $(get_asok_path $the_osd) scrub "$pgid" + + # wait for 'scrubbing' to appear + for ((i=0; i < 80; i++)); do + + st=`ceph pg $pgid query --format json | jq '.state' ` + echo $i ") state now: " $st + + case "$st" in + *scrubbing*repair* ) echo "found scrub+repair"; return 1;; # PR #41258 should have prevented this + *scrubbing* ) echo "found scrub"; return 0;; + *inconsistent* ) echo "Got here too late. Scrub has already finished"; return 1;; + *recovery* ) echo "Got here too late. Scrub has already finished."; return 1;; + * ) echo $st;; + esac + + if [ $((i % 10)) == 4 ]; then + echo "loop --------> " $i + fi + sleep 0.3 + done + + echo "Timeout waiting for deep-scrub of " $pgid " on " $the_osd " to start" + return 1 +} + +function wait_end_of_scrub() { # osd# pg + local the_osd="osd.$1" + local pgid=$2 + + for ((i=0; i < 40; i++)); do + st=`ceph pg $pgid query --format json | jq '.state' ` + echo "wait-scrub-end state now: " $st + [[ $st =~ (.*scrubbing.*) ]] || break + if [ $((i % 5)) == 4 ] ; then + flush_pg_stats + fi + sleep 0.3 + done + + if [[ $st =~ (.*scrubbing.*) ]] + then + # a timeout + return 1 + fi + return 0 +} + + +function TEST_auto_repair_bluestore_tag() { + local dir=$1 + local poolname=testpool + + # Launch a cluster with 3 seconds scrub interval + setup $dir || return 1 + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + local ceph_osd_args="--osd-scrub-auto-repair=true \ + --osd_deep_scrub_randomize_ratio=0 \ + --osd-scrub-interval-randomize-ratio=0" + for id in $(seq 0 2) ; do + run_osd $dir $id $ceph_osd_args || return 1 + done + + create_pool $poolname 1 1 || return 1 + ceph osd pool set $poolname size 2 + wait_for_clean || return 1 + + # Put an object + local payload=ABCDEF + echo $payload > $dir/ORIGINAL + rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1 + + # Remove the object from one shard physically + # Restarted osd get $ceph_osd_args passed + objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1 + + local pgid=$(get_pg $poolname SOMETHING) + local primary=$(get_primary $poolname SOMETHING) + echo "Affected PG " $pgid " w/ primary " $primary + local last_scrub_stamp="$(get_last_scrub_stamp $pgid)" + initiate_and_fetch_state $primary $pgid "3.0" + r=$? + echo "initiate_and_fetch_state ret: " $r + set_config "osd" "$1" "osd_scrub_sleep" "0" + if [ $r -ne 0 ]; then + return 1 + fi + + wait_end_of_scrub "$primary" "$pgid" || return 1 + ceph pg dump pgs + + # Verify - the file should be back + # Restarted osd get $ceph_osd_args passed + objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1 + objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING get-bytes $dir/COPY || return 1 + diff $dir/ORIGINAL $dir/COPY || return 1 + grep scrub_finish $dir/osd.${primary}.log + + # Tear down + teardown $dir || return 1 +} + + function TEST_auto_repair_bluestore_basic() { local dir=$1 local poolname=testpool # Launch a cluster with 5 seconds scrub interval - setup $dir || return 1 run_mon $dir a || return 1 run_mgr $dir x || return 1 local ceph_osd_args="--osd-scrub-auto-repair=true \ - --osd_deep_scrub_randomize_ratio=0 \ + --osd_deep_scrub_randomize_ratio=0 \ --osd-scrub-interval-randomize-ratio=0" for id in $(seq 0 2) ; do run_osd $dir $id $ceph_osd_args || return 1 @@ -373,9 +481,6 @@ function TEST_auto_repair_bluestore_basic() { objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING get-bytes $dir/COPY || return 1 diff $dir/ORIGINAL $dir/COPY || return 1 grep scrub_finish $dir/osd.${primary}.log - - # Tear down - teardown $dir || return 1 } function TEST_auto_repair_bluestore_scrub() { @@ -383,12 +488,12 @@ function TEST_auto_repair_bluestore_scrub() { local poolname=testpool # Launch a cluster with 5 seconds scrub interval - setup $dir || return 1 run_mon $dir a || return 1 run_mgr $dir x || return 1 local ceph_osd_args="--osd-scrub-auto-repair=true \ - --osd_deep_scrub_randomize_ratio=0 \ - --osd-scrub-interval-randomize-ratio=0" + --osd_deep_scrub_randomize_ratio=0 \ + --osd-scrub-interval-randomize-ratio=0 \ + --osd-scrub-backoff-ratio=0" for id in $(seq 0 2) ; do run_osd $dir $id $ceph_osd_args || return 1 done @@ -428,9 +533,6 @@ function TEST_auto_repair_bluestore_scrub() { # This should have caused 1 object to be repaired COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired') test "$COUNT" = "1" || return 1 - - # Tear down - teardown $dir || return 1 } function TEST_auto_repair_bluestore_failed() { @@ -438,11 +540,10 @@ function TEST_auto_repair_bluestore_failed() { local poolname=testpool # Launch a cluster with 5 seconds scrub interval - setup $dir || return 1 run_mon $dir a || return 1 run_mgr $dir x || return 1 local ceph_osd_args="--osd-scrub-auto-repair=true \ - --osd_deep_scrub_randomize_ratio=0 \ + --osd_deep_scrub_randomize_ratio=0 \ --osd-scrub-interval-randomize-ratio=0" for id in $(seq 0 2) ; do run_osd $dir $id $ceph_osd_args || return 1 @@ -498,9 +599,6 @@ function TEST_auto_repair_bluestore_failed() { ceph pg dump pgs ceph pg dump pgs | grep -q -e "^${pgid}.* active+clean " -e "^${pgid}.* active+clean+wait " || return 1 grep scrub_finish $dir/osd.${primary}.log - - # Tear down - teardown $dir || return 1 } function TEST_auto_repair_bluestore_failed_norecov() { @@ -508,11 +606,10 @@ function TEST_auto_repair_bluestore_failed_norecov() { local poolname=testpool # Launch a cluster with 5 seconds scrub interval - setup $dir || return 1 run_mon $dir a || return 1 run_mgr $dir x || return 1 local ceph_osd_args="--osd-scrub-auto-repair=true \ - --osd_deep_scrub_randomize_ratio=0 \ + --osd_deep_scrub_randomize_ratio=0 \ --osd-scrub-interval-randomize-ratio=0" for id in $(seq 0 2) ; do run_osd $dir $id $ceph_osd_args || return 1 @@ -552,9 +649,6 @@ function TEST_auto_repair_bluestore_failed_norecov() { grep -q "scrub_finish.*present with no repair possible" $dir/osd.${primary}.log || return 1 ceph pg dump pgs ceph pg dump pgs | grep -q "^${pgid}.*+failed_repair" || return 1 - - # Tear down - teardown $dir || return 1 } function TEST_repair_stats() { @@ -566,7 +660,6 @@ function TEST_repair_stats() { local REPAIRS=20 # Launch a cluster with 5 seconds scrub interval - setup $dir || return 1 run_mon $dir a || return 1 run_mgr $dir x || return 1 local ceph_osd_args="--osd_deep_scrub_randomize_ratio=0 \ @@ -626,9 +719,6 @@ function TEST_repair_stats() { ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum" COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum.num_shards_repaired") test "$COUNT" = "$REPAIRS" || return 1 - - # Tear down - teardown $dir || return 1 } function TEST_repair_stats_ec() { @@ -641,7 +731,6 @@ function TEST_repair_stats_ec() { local allow_overwrites=false # Launch a cluster with 5 seconds scrub interval - setup $dir || return 1 run_mon $dir a || return 1 run_mgr $dir x || return 1 local ceph_osd_args="--osd_deep_scrub_randomize_ratio=0 \ @@ -704,9 +793,6 @@ function TEST_repair_stats_ec() { ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum" COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum.num_shards_repaired") test "$COUNT" = "$REPAIRS" || return 1 - - # Tear down - teardown $dir || return 1 } function corrupt_and_repair_jerasure() { @@ -714,7 +800,6 @@ function corrupt_and_repair_jerasure() { local allow_overwrites=$2 local poolname=ecpool - setup $dir || return 1 run_mon $dir a || return 1 run_mgr $dir x || return 1 for id in $(seq 0 3) ; do @@ -729,8 +814,6 @@ function corrupt_and_repair_jerasure() { create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1 corrupt_and_repair_erasure_coded $dir $poolname || return 1 - - teardown $dir || return 1 } function TEST_corrupt_and_repair_jerasure_appends() { @@ -748,7 +831,6 @@ function corrupt_and_repair_lrc() { local allow_overwrites=$2 local poolname=ecpool - setup $dir || return 1 run_mon $dir a || return 1 run_mgr $dir x || return 1 for id in $(seq 0 9) ; do @@ -763,8 +845,6 @@ function corrupt_and_repair_lrc() { create_ec_pool $poolname $allow_overwrites k=4 m=2 l=3 plugin=lrc || return 1 corrupt_and_repair_erasure_coded $dir $poolname || return 1 - - teardown $dir || return 1 } function TEST_corrupt_and_repair_lrc_appends() { @@ -783,7 +863,6 @@ function unfound_erasure_coded() { local poolname=ecpool local payload=ABCDEF - setup $dir || return 1 run_mon $dir a || return 1 run_mgr $dir x || return 1 for id in $(seq 0 3) ; do @@ -831,8 +910,6 @@ function unfound_erasure_coded() { ceph -s|grep "4 up" || return 1 ceph -s|grep "4 in" || return 1 ceph -s|grep "1/1 objects unfound" || return 1 - - teardown $dir || return 1 } function TEST_unfound_erasure_coded_appends() { @@ -853,7 +930,6 @@ function list_missing_erasure_coded() { local allow_overwrites=$2 local poolname=ecpool - setup $dir || return 1 run_mon $dir a || return 1 run_mgr $dir x || return 1 for id in $(seq 0 2) ; do @@ -913,8 +989,6 @@ function list_missing_erasure_coded() { matches=$(ceph pg $pg list_unfound | egrep "MOBJ0|MOBJ1" | wc -l) [ $matches -eq 2 ] && break done - - teardown $dir || return 1 } function TEST_list_missing_erasure_coded_appends() { @@ -935,7 +1009,6 @@ function TEST_corrupt_scrub_replicated() { local poolname=csr_pool local total_objs=19 - setup $dir || return 1 run_mon $dir a --osd_pool_default_size=2 || return 1 run_mgr $dir x || return 1 run_osd $dir 0 || return 1 @@ -3530,7 +3603,6 @@ EOF fi ceph osd pool rm $poolname $poolname --yes-i-really-really-mean-it - teardown $dir || return 1 } @@ -3543,7 +3615,6 @@ function corrupt_scrub_erasure() { local poolname=ecpool local total_objs=7 - setup $dir || return 1 run_mon $dir a || return 1 run_mgr $dir x || return 1 for id in $(seq 0 2) ; do @@ -5690,7 +5761,6 @@ EOF fi ceph osd pool rm $poolname $poolname --yes-i-really-really-mean-it - teardown $dir || return 1 } function TEST_corrupt_scrub_erasure_appends() { @@ -5711,7 +5781,6 @@ function TEST_periodic_scrub_replicated() { local poolname=psr_pool local objname=POBJ - setup $dir || return 1 run_mon $dir a --osd_pool_default_size=2 || return 1 run_mgr $dir x || return 1 local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 " @@ -5803,7 +5872,6 @@ function TEST_scrub_warning() { local conf_overdue_seconds=$(calc $i7_days + $i1_day + \( $i7_days \* $overdue \) ) local pool_overdue_seconds=$(calc $i14_days + $i1_day + \( $i14_days \* $overdue \) ) - setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 --mon_allow_pool_size_one=true || return 1 run_mgr $dir x --mon_warn_pg_not_scrubbed_ratio=${overdue} --mon_warn_pg_not_deep_scrubbed_ratio=${overdue} || return 1 run_osd $dir 0 $ceph_osd_args --osd_scrub_backoff_ratio=0 || return 1 @@ -5870,7 +5938,6 @@ function TEST_scrub_warning() { ceph health detail | grep "not deep-scrubbed since" return 1 fi - return 0 } # @@ -5881,7 +5948,6 @@ function TEST_corrupt_snapset_scrub_rep() { local poolname=csr_pool local total_objs=2 - setup $dir || return 1 run_mon $dir a --osd_pool_default_size=2 || return 1 run_mgr $dir x || return 1 run_osd $dir 0 || return 1 @@ -6141,7 +6207,6 @@ EOF fi ceph osd pool rm $poolname $poolname --yes-i-really-really-mean-it - teardown $dir || return 1 } function TEST_request_scrub_priority() { @@ -6151,7 +6216,6 @@ function TEST_request_scrub_priority() { local OBJECTS=64 local PGS=8 - setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 --mon_allow_pool_size_one=true || return 1 run_mgr $dir x || return 1 local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 " @@ -6199,8 +6263,6 @@ function TEST_request_scrub_priority() { # Verify that the requested scrub ran first grep "log_channel.*scrub ok" $dir/osd.${primary}.log | grep -v purged_snaps | head -1 | sed 's/.*[[]DBG[]]//' | grep -q $pg || return 1 - - return 0 } diff --git a/ceph/qa/suites/fs/functional/tasks/mds-full.yaml b/ceph/qa/suites/fs/functional/tasks/mds-full.yaml index 0c8692877..9399890c4 100644 --- a/ceph/qa/suites/fs/functional/tasks/mds-full.yaml +++ b/ceph/qa/suites/fs/functional/tasks/mds-full.yaml @@ -12,6 +12,8 @@ overrides: - is full \(reached quota - POOL_FULL - POOL_BACKFILLFULL + - PG_RECOVERY_FULL + - PG_DEGRADED conf: mon: mon osd nearfull ratio: 0.6 diff --git a/ceph/qa/suites/fs/upgrade/nofs/% b/ceph/qa/suites/fs/upgrade/nofs/% new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/fs/upgrade/nofs/.qa b/ceph/qa/suites/fs/upgrade/nofs/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/fs/upgrade/nofs/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/fs/upgrade/nofs/README b/ceph/qa/suites/fs/upgrade/nofs/README new file mode 100644 index 000000000..e7f6960ef --- /dev/null +++ b/ceph/qa/suites/fs/upgrade/nofs/README @@ -0,0 +1,3 @@ +This test just verifies that upgrades work with no file system present. In +particular, catch that MDSMonitor doesn't blow up somehow with version +mismatches. diff --git a/ceph/qa/suites/fs/upgrade/nofs/bluestore-bitmap.yaml b/ceph/qa/suites/fs/upgrade/nofs/bluestore-bitmap.yaml new file mode 120000 index 000000000..fb603bc9a --- /dev/null +++ b/ceph/qa/suites/fs/upgrade/nofs/bluestore-bitmap.yaml @@ -0,0 +1 @@ +.qa/cephfs/objectstore-ec/bluestore-bitmap.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/upgrade/nofs/centos_latest.yaml b/ceph/qa/suites/fs/upgrade/nofs/centos_latest.yaml new file mode 120000 index 000000000..bd9854e70 --- /dev/null +++ b/ceph/qa/suites/fs/upgrade/nofs/centos_latest.yaml @@ -0,0 +1 @@ +.qa/distros/supported/centos_latest.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/upgrade/nofs/conf b/ceph/qa/suites/fs/upgrade/nofs/conf new file mode 120000 index 000000000..6d4712984 --- /dev/null +++ b/ceph/qa/suites/fs/upgrade/nofs/conf @@ -0,0 +1 @@ +.qa/cephfs/conf/ \ No newline at end of file diff --git a/ceph/qa/suites/fs/upgrade/nofs/no-mds-cluster.yaml b/ceph/qa/suites/fs/upgrade/nofs/no-mds-cluster.yaml new file mode 100644 index 000000000..33c6fb16b --- /dev/null +++ b/ceph/qa/suites/fs/upgrade/nofs/no-mds-cluster.yaml @@ -0,0 +1,6 @@ +roles: +- [mon.a, mon.b, mon.c, mgr.x, mgr.y, osd.0, osd.1, osd.2, osd.3] +openstack: +- volumes: # attached to each instance + count: 4 + size: 10 # GB diff --git a/ceph/qa/suites/fs/upgrade/nofs/overrides/% b/ceph/qa/suites/fs/upgrade/nofs/overrides/% new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/fs/upgrade/nofs/overrides/.qa b/ceph/qa/suites/fs/upgrade/nofs/overrides/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/fs/upgrade/nofs/overrides/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/fs/upgrade/nofs/overrides/pg-warn.yaml b/ceph/qa/suites/fs/upgrade/nofs/overrides/pg-warn.yaml new file mode 100644 index 000000000..4ae54a40d --- /dev/null +++ b/ceph/qa/suites/fs/upgrade/nofs/overrides/pg-warn.yaml @@ -0,0 +1,5 @@ +overrides: + ceph: + conf: + global: + mon pg warn min per osd: 0 diff --git a/ceph/qa/suites/fs/upgrade/nofs/overrides/whitelist_health.yaml b/ceph/qa/suites/fs/upgrade/nofs/overrides/whitelist_health.yaml new file mode 120000 index 000000000..74f39a49b --- /dev/null +++ b/ceph/qa/suites/fs/upgrade/nofs/overrides/whitelist_health.yaml @@ -0,0 +1 @@ +.qa/cephfs/overrides/whitelist_health.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/upgrade/nofs/overrides/whitelist_wrongly_marked_down.yaml b/ceph/qa/suites/fs/upgrade/nofs/overrides/whitelist_wrongly_marked_down.yaml new file mode 120000 index 000000000..b4528c0f8 --- /dev/null +++ b/ceph/qa/suites/fs/upgrade/nofs/overrides/whitelist_wrongly_marked_down.yaml @@ -0,0 +1 @@ +.qa/cephfs/overrides/whitelist_wrongly_marked_down.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/upgrade/nofs/tasks/% b/ceph/qa/suites/fs/upgrade/nofs/tasks/% new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/fs/upgrade/nofs/tasks/.qa b/ceph/qa/suites/fs/upgrade/nofs/tasks/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/fs/upgrade/nofs/tasks/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/fs/upgrade/nofs/tasks/0-octopus.yaml b/ceph/qa/suites/fs/upgrade/nofs/tasks/0-octopus.yaml new file mode 100644 index 000000000..40d34753d --- /dev/null +++ b/ceph/qa/suites/fs/upgrade/nofs/tasks/0-octopus.yaml @@ -0,0 +1,38 @@ +meta: +- desc: | + install ceph/octopus latest +tasks: +- install: + branch: octopus + exclude_packages: + - librados3 + - ceph-mgr-dashboard + - ceph-mgr-diskprediction-local + - ceph-mgr-rook + - ceph-mgr-cephadm + - cephadm + extra_packages: ['librados2'] +- print: "**** done installing octopus" +- ceph: + log-ignorelist: + - overall HEALTH_ + - \(FS_ + - \(MDS_ + - \(OSD_ + - \(MON_DOWN\) + - \(CACHE_POOL_ + - \(POOL_ + - \(MGR_DOWN\) + - \(PG_ + - \(SMALLER_PGP_NUM\) + - Monitor daemon marked osd + - Behind on trimming + - Manager daemon + conf: + global: + mon warn on pool no app: false + ms bind msgr2: false +- exec: + osd.0: + - ceph osd set-require-min-compat-client octopus +- print: "**** done ceph" diff --git a/ceph/qa/suites/fs/upgrade/nofs/tasks/1-upgrade.yaml b/ceph/qa/suites/fs/upgrade/nofs/tasks/1-upgrade.yaml new file mode 100644 index 000000000..c04ee914b --- /dev/null +++ b/ceph/qa/suites/fs/upgrade/nofs/tasks/1-upgrade.yaml @@ -0,0 +1,45 @@ +overrides: + ceph: + log-ignorelist: + - scrub mismatch + - ScrubResult + - wrongly marked + - \(POOL_APP_NOT_ENABLED\) + - \(SLOW_OPS\) + - overall HEALTH_ + - \(MON_MSGR2_NOT_ENABLED\) + - slow request + conf: + global: + bluestore warn on legacy statfs: false + bluestore warn on no per pool omap: false + mon: + mon warn on osd down out interval zero: false + +tasks: +- print: "*** upgrading, no cephfs present" +- exec: + mon.a: + - ceph fs dump +- install.upgrade: + mon.a: +- print: "**** done install.upgrade" +- ceph.restart: + daemons: [mon.*, mgr.*] + mon-health-to-clog: false + wait-for-healthy: false +- ceph.healthy: +- ceph.restart: + daemons: [osd.*] + wait-for-healthy: false + wait-for-osds-up: true +- exec: + mon.a: + - ceph versions + - ceph osd dump -f json-pretty + - ceph fs dump + - ceph osd require-osd-release octopus + - for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done + #- ceph osd set-require-min-compat-client octopus +- ceph.healthy: +- print: "**** done ceph.restart" diff --git a/ceph/qa/suites/orch/cephadm/dashboard/0-distro/centos_8.2_container_tools_3.0.yaml b/ceph/qa/suites/orch/cephadm/dashboard/0-distro/centos_8.2_container_tools_3.0.yaml new file mode 120000 index 000000000..d1965f3d6 --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/dashboard/0-distro/centos_8.2_container_tools_3.0.yaml @@ -0,0 +1 @@ +.qa/distros/podman/centos_8.2_container_tools_3.0.yaml \ No newline at end of file diff --git a/ceph/qa/suites/orch/cephadm/dashboard/0-distro/centos_8.2_kubic_stable.yaml b/ceph/qa/suites/orch/cephadm/dashboard/0-distro/centos_8.2_kubic_stable.yaml deleted file mode 120000 index 3afeed74d..000000000 --- a/ceph/qa/suites/orch/cephadm/dashboard/0-distro/centos_8.2_kubic_stable.yaml +++ /dev/null @@ -1 +0,0 @@ -.qa/distros/podman/centos_8.2_kubic_stable.yaml \ No newline at end of file diff --git a/ceph/qa/suites/orch/cephadm/smoke/distro/centos_8.2_kubic_stable.yaml b/ceph/qa/suites/orch/cephadm/smoke/distro/centos_8.2_kubic_stable.yaml deleted file mode 120000 index 3afeed74d..000000000 --- a/ceph/qa/suites/orch/cephadm/smoke/distro/centos_8.2_kubic_stable.yaml +++ /dev/null @@ -1 +0,0 @@ -.qa/distros/podman/centos_8.2_kubic_stable.yaml \ No newline at end of file diff --git a/ceph/qa/suites/orch/cephadm/smoke/distro/centos_8.3_container_tools_3.0.yaml b/ceph/qa/suites/orch/cephadm/smoke/distro/centos_8.3_container_tools_3.0.yaml new file mode 120000 index 000000000..479a5c26e --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/smoke/distro/centos_8.3_container_tools_3.0.yaml @@ -0,0 +1 @@ +.qa/distros/podman/centos_8.3_container_tools_3.0.yaml \ No newline at end of file diff --git a/ceph/qa/suites/orch/cephadm/smoke/distro/rhel_8.3_kubic_stable.yaml b/ceph/qa/suites/orch/cephadm/smoke/distro/rhel_8.3_kubic_stable.yaml deleted file mode 120000 index 20f0f7c55..000000000 --- a/ceph/qa/suites/orch/cephadm/smoke/distro/rhel_8.3_kubic_stable.yaml +++ /dev/null @@ -1 +0,0 @@ -.qa/distros/podman/rhel_8.3_kubic_stable.yaml \ No newline at end of file diff --git a/ceph/qa/suites/orch/cephadm/workunits/0-distro/centos_8.2_container_tools_3.0.yaml b/ceph/qa/suites/orch/cephadm/workunits/0-distro/centos_8.2_container_tools_3.0.yaml new file mode 120000 index 000000000..d1965f3d6 --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/workunits/0-distro/centos_8.2_container_tools_3.0.yaml @@ -0,0 +1 @@ +.qa/distros/podman/centos_8.2_container_tools_3.0.yaml \ No newline at end of file diff --git a/ceph/qa/suites/orch/cephadm/workunits/0-distro/centos_8.2_kubic_stable.yaml b/ceph/qa/suites/orch/cephadm/workunits/0-distro/centos_8.2_kubic_stable.yaml deleted file mode 120000 index 3afeed74d..000000000 --- a/ceph/qa/suites/orch/cephadm/workunits/0-distro/centos_8.2_kubic_stable.yaml +++ /dev/null @@ -1 +0,0 @@ -.qa/distros/podman/centos_8.2_kubic_stable.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rados/dashboard/centos_8.2_container_tools_3.0.yaml b/ceph/qa/suites/rados/dashboard/centos_8.2_container_tools_3.0.yaml new file mode 120000 index 000000000..d1965f3d6 --- /dev/null +++ b/ceph/qa/suites/rados/dashboard/centos_8.2_container_tools_3.0.yaml @@ -0,0 +1 @@ +.qa/distros/podman/centos_8.2_container_tools_3.0.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rados/dashboard/tasks/dashboard.yaml b/ceph/qa/suites/rados/dashboard/tasks/dashboard.yaml index 111c5f38e..5ca0c6621 100644 --- a/ceph/qa/suites/rados/dashboard/tasks/dashboard.yaml +++ b/ceph/qa/suites/rados/dashboard/tasks/dashboard.yaml @@ -57,3 +57,4 @@ tasks: - tasks.mgr.dashboard.test_summary - tasks.mgr.dashboard.test_telemetry - tasks.mgr.dashboard.test_user + - tasks.mgr.dashboard.test_motd diff --git a/ceph/qa/suites/rgw/crypt/ignore-pg-availability.yaml b/ceph/qa/suites/rgw/crypt/ignore-pg-availability.yaml index 9889fd2d2..32340b1fa 120000 --- a/ceph/qa/suites/rgw/crypt/ignore-pg-availability.yaml +++ b/ceph/qa/suites/rgw/crypt/ignore-pg-availability.yaml @@ -1 +1 @@ -../ignore-pg-availability.yaml \ No newline at end of file +.qa/rgw/ignore-pg-availability.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rgw/multifs/ignore-pg-availability.yaml b/ceph/qa/suites/rgw/multifs/ignore-pg-availability.yaml index 9889fd2d2..32340b1fa 120000 --- a/ceph/qa/suites/rgw/multifs/ignore-pg-availability.yaml +++ b/ceph/qa/suites/rgw/multifs/ignore-pg-availability.yaml @@ -1 +1 @@ -../ignore-pg-availability.yaml \ No newline at end of file +.qa/rgw/ignore-pg-availability.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rgw/multisite/ignore-pg-availability.yaml b/ceph/qa/suites/rgw/multisite/ignore-pg-availability.yaml index 9889fd2d2..32340b1fa 120000 --- a/ceph/qa/suites/rgw/multisite/ignore-pg-availability.yaml +++ b/ceph/qa/suites/rgw/multisite/ignore-pg-availability.yaml @@ -1 +1 @@ -../ignore-pg-availability.yaml \ No newline at end of file +.qa/rgw/ignore-pg-availability.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rgw/multisite/overrides.yaml b/ceph/qa/suites/rgw/multisite/overrides.yaml index c9019e154..7fff1f49e 100644 --- a/ceph/qa/suites/rgw/multisite/overrides.yaml +++ b/ceph/qa/suites/rgw/multisite/overrides.yaml @@ -14,5 +14,6 @@ overrides: rgw md log max shards: 4 rgw data log num shards: 4 rgw sync obj etag verify: true + rgw sync meta inject err probability: 0.1 rgw: compression type: random diff --git a/ceph/qa/suites/rgw/sts/.qa b/ceph/qa/suites/rgw/sts/.qa new file mode 120000 index 000000000..fea2489fd --- /dev/null +++ b/ceph/qa/suites/rgw/sts/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/ceph/qa/suites/rgw/sts/ignore-pg-availability.yaml b/ceph/qa/suites/rgw/sts/ignore-pg-availability.yaml index 9889fd2d2..32340b1fa 120000 --- a/ceph/qa/suites/rgw/sts/ignore-pg-availability.yaml +++ b/ceph/qa/suites/rgw/sts/ignore-pg-availability.yaml @@ -1 +1 @@ -../ignore-pg-availability.yaml \ No newline at end of file +.qa/rgw/ignore-pg-availability.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rgw/verify/ignore-pg-availability.yaml b/ceph/qa/suites/rgw/verify/ignore-pg-availability.yaml index 9889fd2d2..32340b1fa 120000 --- a/ceph/qa/suites/rgw/verify/ignore-pg-availability.yaml +++ b/ceph/qa/suites/rgw/verify/ignore-pg-availability.yaml @@ -1 +1 @@ -../ignore-pg-availability.yaml \ No newline at end of file +.qa/rgw/ignore-pg-availability.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-parallel/point-to-point-upgrade.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-parallel/point-to-point-upgrade.yaml index a433f607e..9a2d9d8c8 100644 --- a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-parallel/point-to-point-upgrade.yaml +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-parallel/point-to-point-upgrade.yaml @@ -3,7 +3,7 @@ meta: Run ceph on two nodes, using one of them as a client, with a separate client-only node. Use xfs beneath the osds. - install ceph/pacific v16.2.2 and the v16.2.x point versions + install ceph/pacific v16.2.4 and the v16.2.x point versions run workload and upgrade-sequence in parallel (every point release should be tested) run workload and upgrade-sequence in parallel @@ -69,32 +69,32 @@ openstack: count: 3 size: 30 # GB tasks: -- print: "**** done pacific v16.2.0 about to install" +- print: "**** done pacific about to install v16.2.4 " - install: - tag: v16.2.2 + tag: v16.2.4 # line below can be removed its from jewel test #exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev', 'librgw2'] -- print: "**** done v16.2.2 install" +- print: "**** done v16.2.4 install" - ceph: fs: xfs add_osds_to_crush: true - print: "**** done ceph xfs" - sequential: - workload -- print: "**** done workload v16.2.2" +- print: "**** done workload v16.2.4" -####### upgrade to v16.2.3 +####### upgrade to v16.2.5 - install.upgrade: #exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev'] mon.a: - tag: v16.2.3 + tag: v16.2.5 mon.b: - tag: v16.2.3 + tag: v16.2.5 - parallel: - workload_pacific - upgrade-sequence_pacific -- print: "**** done parallel pacific v16.2.3" +- print: "**** done parallel pacific v16.2.5" #### upgrade to latest pacific - install.upgrade: diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/1-ceph-install/pacific..yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/1-ceph-install/pacific..yaml index 2de7badc4..537ed596e 100644 --- a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/1-ceph-install/pacific..yaml +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/1-ceph-install/pacific..yaml @@ -1,13 +1,13 @@ meta: - desc: | - install ceph/pacific v16.2.3 + install ceph/pacific v16.2.4 Overall upgrade path is - pacific-latest.point => pacific-latest tasks: - install: - tag: v16.2.3 + tag: v16.2.4 exclude_packages: ['librados3'] extra_packages: ['librados2'] -- print: "**** done install pacific v16.2.3" +- print: "**** done install pacific v16.2.4" - ceph: - exec: osd.0: diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/6-final-workload/rbd-python.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/6-final-workload/rbd-python.yaml index 4578b4dac..6436757c5 100644 --- a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/6-final-workload/rbd-python.yaml +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/6-final-workload/rbd-python.yaml @@ -3,7 +3,7 @@ meta: librbd python api tests tasks: - workunit: - tag: v16.2.0 + tag: v16.2.4 clients: client.0: - rbd/test_librbd_python.sh diff --git a/ceph/qa/tasks/ceph_manager.py b/ceph/qa/tasks/ceph_manager.py index 1d1f9049e..28f28f54f 100644 --- a/ceph/qa/tasks/ceph_manager.py +++ b/ceph/qa/tasks/ceph_manager.py @@ -3,6 +3,7 @@ ceph manager -- Thrasher and CephManager objects """ from functools import wraps import contextlib +import errno import random import signal import time @@ -3078,13 +3079,22 @@ class CephManager: Loop until quorum size is reached. """ self.log('waiting for quorum size %d' % size) - start = time.time() - while not len(self.get_mon_quorum()) == size: - if timeout is not None: - assert time.time() - start < timeout, \ - ('failed to reach quorum size %d ' - 'before timeout expired' % size) - time.sleep(3) + sleep = 3 + with safe_while(sleep=sleep, + tries=timeout // sleep, + action=f'wait for quorum size {size}') as proceed: + while proceed(): + try: + if len(self.get_mon_quorum()) == size: + break + except CommandFailedError as e: + # could fail instea4d of blocked if the rotating key of the + # connected monitor is not updated yet after they form the + # quorum + if e.exitstatus == errno.EACCES: + pass + else: + raise self.log("quorum is size %d" % size) def get_mon_health(self, debug=False): diff --git a/ceph/qa/tasks/cephfs/caps_helper.py b/ceph/qa/tasks/cephfs/caps_helper.py index a66333546..39b5963be 100644 --- a/ceph/qa/tasks/cephfs/caps_helper.py +++ b/ceph/qa/tasks/cephfs/caps_helper.py @@ -60,7 +60,7 @@ class CapsHelper(CephFSTestCase): self.assertEqual(data, contents1) def conduct_neg_test_for_write_caps(self, filepaths, mounts): - cmdargs = ['echo', 'some random data', Raw('|'), 'sudo', 'tee'] + cmdargs = ['echo', 'some random data', Raw('|'), 'tee'] for mount in mounts: for path in filepaths: diff --git a/ceph/qa/tasks/cephfs/cephfs_test_case.py b/ceph/qa/tasks/cephfs/cephfs_test_case.py index 9ef7989d9..40af40625 100644 --- a/ceph/qa/tasks/cephfs/cephfs_test_case.py +++ b/ceph/qa/tasks/cephfs/cephfs_test_case.py @@ -4,12 +4,10 @@ import os import re from shlex import split as shlex_split -from io import StringIO from tasks.ceph_test_case import CephTestCase from teuthology import contextutil -from teuthology.misc import sudo_write_file from teuthology.orchestra import run from teuthology.orchestra.run import CommandFailedError @@ -445,9 +443,7 @@ class CephFSTestCase(CephTestCase): return self.run_cluster_cmd(f'auth get {self.client_name}') def create_keyring_file(self, remote, keyring): - keyring_path = remote.run(args=['mktemp'], stdout=StringIO()).\ - stdout.getvalue().strip() - sudo_write_file(remote, keyring_path, keyring) + keyring_path = remote.mktemp(data=keyring) # required when triggered using vstart_runner.py. remote.run(args=['chmod', '644', keyring_path]) diff --git a/ceph/qa/tasks/cephfs/filesystem.py b/ceph/qa/tasks/cephfs/filesystem.py index 34ee42f13..e66185c42 100644 --- a/ceph/qa/tasks/cephfs/filesystem.py +++ b/ceph/qa/tasks/cephfs/filesystem.py @@ -71,9 +71,12 @@ class FSStatus(object): """ Operations on a snapshot of the FSMap. """ - def __init__(self, mon_manager): + def __init__(self, mon_manager, epoch=None): self.mon = mon_manager - self.map = json.loads(self.mon.raw_cluster_cmd("fs", "dump", "--format=json")) + cmd = ["fs", "dump", "--format=json"] + if epoch is not None: + cmd.append(str(epoch)) + self.map = json.loads(self.mon.raw_cluster_cmd(*cmd)) def __str__(self): return json.dumps(self.map, indent = 2, sort_keys = True) @@ -368,8 +371,8 @@ class MDSCluster(CephCluster): def newfs(self, name='cephfs', create=True): return Filesystem(self._ctx, name=name, create=create) - def status(self): - return FSStatus(self.mon_manager) + def status(self, epoch=None): + return FSStatus(self.mon_manager, epoch) def get_standby_daemons(self): return set([s['name'] for s in self.status().get_standbys()]) @@ -611,13 +614,34 @@ class Filesystem(MDSCluster): def set_allow_new_snaps(self, yes): self.set_var("allow_new_snaps", yes, '--yes-i-really-mean-it') + def compat(self, *args): + a = map(lambda x: str(x).lower(), args) + self.mon_manager.raw_cluster_cmd("fs", "compat", self.name, *a) + + def add_compat(self, *args): + self.compat("add_compat", *args) + + def add_incompat(self, *args): + self.compat("add_incompat", *args) + + def rm_compat(self, *args): + self.compat("rm_compat", *args) + + def rm_incompat(self, *args): + self.compat("rm_incompat", *args) + def required_client_features(self, *args, **kwargs): c = ["fs", "required_client_features", self.name, *args] return self.mon_manager.run_cluster_cmd(args=c, **kwargs) - # In Octopus+, the PG count can be omitted to use the default. We keep the - # hard-coded value for deployments of Mimic/Nautilus. - pgs_per_fs_pool = 8 + # Since v15.1.0 the pg autoscale mode has been enabled as default, + # will let the pg autoscale mode to calculate the pg_num as needed. + # We set the pg_num_min to 64 to make sure that pg autoscale mode + # won't set the pg_num to low to fix Tracker#45434. + pg_num = 64 + pg_num_min = 64 + target_size_ratio = 0.9 + target_size_ratio_ec = 0.9 def create(self): if self.name is None: @@ -629,13 +653,22 @@ class Filesystem(MDSCluster): else: data_pool_name = self.data_pool_name + # will use the ec pool to store the data and a small amount of + # metadata still goes to the primary data pool for all files. + if not self.metadata_overlay and self.ec_profile and 'disabled' not in self.ec_profile: + self.target_size_ratio = 0.05 + log.debug("Creating filesystem '{0}'".format(self.name)) self.mon_manager.raw_cluster_cmd('osd', 'pool', 'create', - self.metadata_pool_name, self.pgs_per_fs_pool.__str__()) + self.metadata_pool_name, str(self.pg_num), + '--pg_num_min', str(self.pg_num_min)) self.mon_manager.raw_cluster_cmd('osd', 'pool', 'create', - data_pool_name, self.pgs_per_fs_pool.__str__()) + data_pool_name, str(self.pg_num), + '--pg_num_min', str(self.pg_num_min), + '--target_size_ratio', + str(self.target_size_ratio)) if self.metadata_overlay: self.mon_manager.raw_cluster_cmd('fs', 'new', @@ -654,9 +687,10 @@ class Filesystem(MDSCluster): cmd.extend(self.ec_profile) self.mon_manager.raw_cluster_cmd(*cmd) self.mon_manager.raw_cluster_cmd( - 'osd', 'pool', 'create', - ec_data_pool_name, self.pgs_per_fs_pool.__str__(), 'erasure', - ec_data_pool_name) + 'osd', 'pool', 'create', ec_data_pool_name, + 'erasure', ec_data_pool_name, + '--pg_num_min', str(self.pg_num_min), + '--target_size_ratio', str(self.target_size_ratio_ec)) self.mon_manager.raw_cluster_cmd( 'osd', 'pool', 'set', ec_data_pool_name, 'allow_ec_overwrites', 'true') @@ -693,12 +727,15 @@ class Filesystem(MDSCluster): self.getinfo(refresh = True) + # wait pgs to be clean + self.mon_manager.wait_for_clean() + def run_client_payload(self, cmd): # avoid circular dep by importing here: from tasks.cephfs.fuse_mount import FuseMount d = misc.get_testdir(self._ctx) m = FuseMount(self._ctx, {}, d, "admin", self.client_remote, cephfs_name=self.name) - m.mount() + m.mount_wait() m.run_shell_payload(cmd) m.umount_wait(require_clean=True) @@ -810,7 +847,8 @@ class Filesystem(MDSCluster): def add_data_pool(self, name, create=True): if create: - self.mon_manager.raw_cluster_cmd('osd', 'pool', 'create', name, self.pgs_per_fs_pool.__str__()) + self.mon_manager.raw_cluster_cmd('osd', 'pool', 'create', name, + '--pg_num_min', str(self.pg_num_min)) self.mon_manager.raw_cluster_cmd('fs', 'add_data_pool', self.name, name) self.get_pool_names(refresh = True) for poolid, fs_name in self.data_pools.items(): @@ -863,6 +901,12 @@ class Filesystem(MDSCluster): raise RuntimeError("can't set filesystem name if its fscid is set") self.data_pool_name = name + def get_pool_pg_num(self, pool_name): + pgs = json.loads(self.mon_manager.raw_cluster_cmd('osd', 'pool', 'get', + pool_name, 'pg_num', + '--format=json-pretty')) + return int(pgs['pg_num']) + def get_namespace_id(self): return self.id diff --git a/ceph/qa/tasks/cephfs/fuse_mount.py b/ceph/qa/tasks/cephfs/fuse_mount.py index 29dcc6d3b..5c5d1c85c 100644 --- a/ceph/qa/tasks/cephfs/fuse_mount.py +++ b/ceph/qa/tasks/cephfs/fuse_mount.py @@ -446,7 +446,7 @@ print(_find_admin_socket("{client_name}")) client_name="client.{0}".format(self.client_id), mountpoint=self.mountpoint) - asok_path = self.run_python(pyscript) + asok_path = self.run_python(pyscript, sudo=True) log.info("Found client admin socket at {0}".format(asok_path)) return asok_path diff --git a/ceph/qa/tasks/cephfs/kernel_mount.py b/ceph/qa/tasks/cephfs/kernel_mount.py index 984545419..e1b8714e8 100644 --- a/ceph/qa/tasks/cephfs/kernel_mount.py +++ b/ceph/qa/tasks/cephfs/kernel_mount.py @@ -1,5 +1,6 @@ import json import logging +import os import re from io import StringIO @@ -165,64 +166,76 @@ class KernelMount(CephFSMount): if self.mounted: self.umount() - def _find_debug_dir(self): + def _get_debug_dir(self): """ - Find the debugfs folder for this mount + Get the debugfs folder for this mount """ - pyscript = dedent(""" - import glob - import os - import json - def get_id_to_dir(): - result = {} - for dir in glob.glob("/sys/kernel/debug/ceph/*"): - mds_sessions_lines = open(os.path.join(dir, "mds_sessions")).readlines() - client_id = mds_sessions_lines[1].split()[1].strip('"') + cluster_name = 'ceph' + fsid = self.ctx.ceph[cluster_name].fsid - result[client_id] = dir - return result + global_id = self._get_global_id() - print(json.dumps(get_id_to_dir())) - """) - - output = self.client_remote.sh([ - 'sudo', 'python3', '-c', pyscript - ], timeout=(5*60)) - client_id_to_dir = json.loads(output) - - try: - return client_id_to_dir[self.client_id] - except KeyError: - log.error("Client id '{0}' debug dir not found (clients seen were: {1})".format( - self.client_id, ",".join(client_id_to_dir.keys()) - )) - raise + return os.path.join("/sys/kernel/debug/ceph/", f"{fsid}.client{global_id}") def read_debug_file(self, filename): """ Read the debug file "filename", return None if the file doesn't exist. """ - debug_dir = self._find_debug_dir() - pyscript = dedent(""" - import os - - print(open(os.path.join("{debug_dir}", "{filename}")).read()) - """).format(debug_dir=debug_dir, filename=filename) + path = os.path.join(self._get_debug_dir(), filename) + stdout = StringIO() stderr = StringIO() try: - output = self.client_remote.sh([ - 'sudo', 'python3', '-c', pyscript - ], stderr=stderr, timeout=(5*60)) - - return output + self.run_shell_payload(f"sudo dd if={path}", timeout=(5*60), + stdout=stdout, stderr=stderr) + return stdout.getvalue() except CommandFailedError: if 'no such file or directory' in stderr.getvalue().lower(): return None raise + def _get_global_id(self): + try: + p = self.run_shell_payload("getfattr --only-values -n ceph.client_id .", stdout=StringIO()) + v = p.stdout.getvalue() + prefix = "client" + assert v.startswith(prefix) + return int(v[len(prefix):]) + except CommandFailedError: + # Probably this fallback can be deleted in a few releases when the kernel xattr is widely available. + log.debug("Falling back to messy global_id lookup via /sys...") + + pyscript = dedent(""" + import glob + import os + import json + + def get_id_to_dir(): + result = {} + for dir in glob.glob("/sys/kernel/debug/ceph/*"): + mds_sessions_lines = open(os.path.join(dir, "mds_sessions")).readlines() + global_id = mds_sessions_lines[0].split()[1].strip('"') + client_id = mds_sessions_lines[1].split()[1].strip('"') + result[client_id] = global_id + return result + print(json.dumps(get_id_to_dir())) + """) + + output = self.client_remote.sh([ + 'sudo', 'python3', '-c', pyscript + ], timeout=(5*60)) + client_id_to_global_id = json.loads(output) + + try: + return client_id_to_global_id[self.client_id] + except KeyError: + log.error("Client id '{0}' debug dir not found (clients seen were: {1})".format( + self.client_id, ",".join(client_id_to_global_id.keys()) + )) + raise + def get_global_id(self): """ Look up the CephFS client ID for this mount, using debugfs. @@ -230,11 +243,7 @@ class KernelMount(CephFSMount): assert self.mounted - mds_sessions = self.read_debug_file("mds_sessions") - assert mds_sessions - - lines = mds_sessions.split("\n") - return int(lines[0].split()[1]) + return self._get_global_id() @property def _global_addr(self): diff --git a/ceph/qa/tasks/cephfs/mount.py b/ceph/qa/tasks/cephfs/mount.py index d7e775f64..883acb4d8 100644 --- a/ceph/qa/tasks/cephfs/mount.py +++ b/ceph/qa/tasks/cephfs/mount.py @@ -12,7 +12,7 @@ from textwrap import dedent from IPy import IP from teuthology.contextutil import safe_while -from teuthology.misc import get_file, sudo_write_file +from teuthology.misc import get_file, write_file from teuthology.orchestra import run from teuthology.orchestra.run import CommandFailedError, ConnectionLostError, Raw @@ -590,7 +590,7 @@ class CephFSMount(object): for suffix in self.test_files: log.info("Creating file {0}".format(suffix)) self.client_remote.run(args=[ - 'sudo', 'touch', os.path.join(self.hostfs_mntpt, suffix) + 'touch', os.path.join(self.hostfs_mntpt, suffix) ]) def test_create_file(self, filename='testfile', dirname=None, user=None, @@ -604,7 +604,7 @@ class CephFSMount(object): for suffix in self.test_files: log.info("Checking file {0}".format(suffix)) r = self.client_remote.run(args=[ - 'sudo', 'ls', os.path.join(self.hostfs_mntpt, suffix) + 'ls', os.path.join(self.hostfs_mntpt, suffix) ], check_status=False) if r.exitstatus != 0: raise RuntimeError("Expected file {0} not found".format(suffix)) @@ -617,7 +617,7 @@ class CephFSMount(object): if path.find(self.hostfs_mntpt) == -1: path = os.path.join(self.hostfs_mntpt, path) - sudo_write_file(self.client_remote, path, data) + write_file(self.client_remote, path, data) if perms: self.run_shell(args=f'chmod {perms} {path}') @@ -629,7 +629,7 @@ class CephFSMount(object): if path.find(self.hostfs_mntpt) == -1: path = os.path.join(self.hostfs_mntpt, path) - return self.run_shell(args=['sudo', 'cat', path], omit_sudo=False).\ + return self.run_shell(args=['cat', path]).\ stdout.getvalue().strip() def create_destroy(self): @@ -638,34 +638,36 @@ class CephFSMount(object): filename = "{0} {1}".format(datetime.datetime.now(), self.client_id) log.debug("Creating test file {0}".format(filename)) self.client_remote.run(args=[ - 'sudo', 'touch', os.path.join(self.hostfs_mntpt, filename) + 'touch', os.path.join(self.hostfs_mntpt, filename) ]) log.debug("Deleting test file {0}".format(filename)) self.client_remote.run(args=[ - 'sudo', 'rm', '-f', os.path.join(self.hostfs_mntpt, filename) + 'rm', '-f', os.path.join(self.hostfs_mntpt, filename) ]) - def _run_python(self, pyscript, py_version='python3'): - return self.client_remote.run( - args=['sudo', 'adjust-ulimits', 'daemon-helper', 'kill', - py_version, '-c', pyscript], wait=False, stdin=run.PIPE, - stdout=StringIO()) + def _run_python(self, pyscript, py_version='python3', sudo=False): + args = [] + if sudo: + args.append('sudo') + args += ['adjust-ulimits', 'daemon-helper', 'kill', py_version, '-c', pyscript] + return self.client_remote.run(args=args, wait=False, stdin=run.PIPE, stdout=StringIO()) - def run_python(self, pyscript, py_version='python3'): - p = self._run_python(pyscript, py_version) + def run_python(self, pyscript, py_version='python3', sudo=False): + p = self._run_python(pyscript, py_version, sudo=sudo) p.wait() return p.stdout.getvalue().strip() - def run_shell(self, args, omit_sudo=True, timeout=900, **kwargs): + def run_shell(self, args, timeout=900, **kwargs): args = args.split() if isinstance(args, str) else args - # XXX: all commands ran with CephFS mount as CWD must be executed with - # superuser privileges when tests are being run using teuthology. - if args[0] != 'sudo': - args.insert(0, 'sudo') + kwargs.pop('omit_sudo', False) + sudo = kwargs.pop('sudo', False) cwd = kwargs.pop('cwd', self.mountpoint) stdout = kwargs.pop('stdout', StringIO()) stderr = kwargs.pop('stderr', StringIO()) + if sudo: + args.insert(0, 'sudo') + return self.client_remote.run(args=args, cwd=cwd, timeout=timeout, stdout=stdout, stderr=stderr, **kwargs) def run_shell_payload(self, payload, **kwargs): @@ -810,7 +812,7 @@ class CephFSMount(object): i = 0 while i < timeout: r = self.client_remote.run(args=[ - 'sudo', 'ls', os.path.join(self.hostfs_mntpt, basename) + 'stat', os.path.join(self.hostfs_mntpt, basename) ], check_status=False) if r.exitstatus == 0: log.debug("File {0} became visible from {1} after {2}s".format( @@ -908,7 +910,7 @@ class CephFSMount(object): log.info("check lock on file {0}".format(basename)) self.client_remote.run(args=[ - 'sudo', 'python3', '-c', pyscript + 'python3', '-c', pyscript ]) def write_background(self, basename="background_file", loop=False): @@ -969,6 +971,7 @@ class CephFSMount(object): def validate_test_pattern(self, filename, size): log.info("Validating {0} bytes from {1}".format(size, filename)) + # Use sudo because cephfs-data-scan may recreate the file with owner==root return self.run_python(dedent(""" import zlib path = "{path}" @@ -985,7 +988,7 @@ class CephFSMount(object): """.format( path=os.path.join(self.hostfs_mntpt, filename), size=size - ))) + )), sudo=True) def open_n_background(self, fs_path, count): """ @@ -1099,7 +1102,7 @@ class CephFSMount(object): def lstat(self, fs_path, follow_symlinks=False, wait=True): return self.stat(fs_path, follow_symlinks=False, wait=True) - def stat(self, fs_path, follow_symlinks=True, wait=True): + def stat(self, fs_path, follow_symlinks=True, wait=True, **kwargs): """ stat a file, and return the result as a dictionary like this: { @@ -1139,7 +1142,7 @@ class CephFSMount(object): dict([(a, getattr(s, a)) for a in attrs]), indent=2)) """).format(stat_call=stat_call) - proc = self._run_python(pyscript) + proc = self._run_python(pyscript, **kwargs) if wait: proc.wait() return json.loads(proc.stdout.getvalue().strip()) @@ -1205,7 +1208,7 @@ class CephFSMount(object): proc.wait() return int(proc.stdout.getvalue().strip()) - def ls(self, path=None): + def ls(self, path=None, **kwargs): """ Wrap ls: return a list of strings """ @@ -1213,7 +1216,7 @@ class CephFSMount(object): if path: cmd.append(path) - ls_text = self.run_shell(cmd).stdout.getvalue().strip() + ls_text = self.run_shell(cmd, **kwargs).stdout.getvalue().strip() if ls_text: return ls_text.split("\n") @@ -1222,7 +1225,7 @@ class CephFSMount(object): # gives you [''] instead of [] return [] - def setfattr(self, path, key, val): + def setfattr(self, path, key, val, **kwargs): """ Wrap setfattr. @@ -1231,16 +1234,16 @@ class CephFSMount(object): :param val: xattr value :return: None """ - self.run_shell(["setfattr", "-n", key, "-v", val, path]) + self.run_shell(["setfattr", "-n", key, "-v", val, path], **kwargs) - def getfattr(self, path, attr): + def getfattr(self, path, attr, **kwargs): """ Wrap getfattr: return the values of a named xattr on one file, or None if the attribute is not found. :return: a string """ - p = self.run_shell(["getfattr", "--only-values", "-n", attr, path], wait=False) + p = self.run_shell(["getfattr", "--only-values", "-n", attr, path], wait=False, **kwargs) try: p.wait() except CommandFailedError as e: diff --git a/ceph/qa/tasks/cephfs/test_admin.py b/ceph/qa/tasks/cephfs/test_admin.py index c10b9a482..e9bb48718 100644 --- a/ceph/qa/tasks/cephfs/test_admin.py +++ b/ceph/qa/tasks/cephfs/test_admin.py @@ -1,4 +1,7 @@ +import errno import json +import logging +import time import uuid from io import StringIO from os.path import join as os_path_join @@ -10,6 +13,7 @@ from tasks.cephfs.filesystem import FileLayout, FSMissing from tasks.cephfs.fuse_mount import FuseMount from tasks.cephfs.caps_helper import CapsHelper +log = logging.getLogger(__name__) class TestAdminCommands(CephFSTestCase): """ @@ -17,7 +21,7 @@ class TestAdminCommands(CephFSTestCase): """ CLIENTS_REQUIRED = 1 - MDSS_REQUIRED = 1 + MDSS_REQUIRED = 3 def test_fsnames_can_only_by_goodchars(self): n = 'test_fsnames_can_only_by_goodchars' @@ -84,7 +88,8 @@ class TestAdminCommands(CephFSTestCase): """ pool_name = "foo" mon_cmd = self.fs.mon_manager.raw_cluster_cmd - mon_cmd('osd', 'pool', 'create', pool_name, str(self.fs.pgs_per_fs_pool)) + mon_cmd('osd', 'pool', 'create', pool_name, '--pg_num_min', + str(self.fs.pg_num_min)) # Check whether https://tracker.ceph.com/issues/43061 is fixed mon_cmd('osd', 'pool', 'application', 'enable', pool_name, 'cephfs') self.fs.add_data_pool(pool_name, create=False) @@ -187,13 +192,150 @@ class TestAdminCommands(CephFSTestCase): pool_names = [fs_name+'-'+key for key in keys] mon_cmd = self.fs.mon_manager.raw_cluster_cmd for p in pool_names: - mon_cmd('osd', 'pool', 'create', p, str(self.fs.pgs_per_fs_pool)) + mon_cmd('osd', 'pool', 'create', p, '--pg_num_min', str(self.fs.pg_num_min)) mon_cmd('osd', 'pool', 'application', 'enable', p, 'cephfs') mon_cmd('fs', 'new', fs_name, pool_names[0], pool_names[1]) for i in range(2): self._check_pool_application_metadata_key_value( pool_names[i], 'cephfs', keys[i], fs_name) + def test_fs_new_with_specific_id(self): + """ + That a file system can be created with a specific ID. + """ + fs_name = "test_fs_specific_id" + fscid = 100 + keys = ['metadata', 'data'] + pool_names = [fs_name+'-'+key for key in keys] + for p in pool_names: + self.run_cluster_cmd(f'osd pool create {p}') + self.run_cluster_cmd(f'fs new {fs_name} {pool_names[0]} {pool_names[1]} --fscid {fscid} --force') + self.fs.status().get_fsmap(fscid) + for i in range(2): + self._check_pool_application_metadata_key_value(pool_names[i], 'cephfs', keys[i], fs_name) + + def test_fs_new_with_specific_id_idempotency(self): + """ + That command to create file system with specific ID is idempotent. + """ + fs_name = "test_fs_specific_id" + fscid = 100 + keys = ['metadata', 'data'] + pool_names = [fs_name+'-'+key for key in keys] + for p in pool_names: + self.run_cluster_cmd(f'osd pool create {p}') + self.run_cluster_cmd(f'fs new {fs_name} {pool_names[0]} {pool_names[1]} --fscid {fscid} --force') + self.run_cluster_cmd(f'fs new {fs_name} {pool_names[0]} {pool_names[1]} --fscid {fscid} --force') + self.fs.status().get_fsmap(fscid) + + def test_fs_new_with_specific_id_fails_without_force_flag(self): + """ + That command to create file system with specific ID fails without '--force' flag. + """ + fs_name = "test_fs_specific_id" + fscid = 100 + keys = ['metadata', 'data'] + pool_names = [fs_name+'-'+key for key in keys] + for p in pool_names: + self.run_cluster_cmd(f'osd pool create {p}') + try: + self.run_cluster_cmd(f'fs new {fs_name} {pool_names[0]} {pool_names[1]} --fscid {fscid}') + except CommandFailedError as ce: + self.assertEqual(ce.exitstatus, errno.EINVAL, + "invalid error code on creating a file system with specifc ID without --force flag") + else: + self.fail("expected creating file system with specific ID without '--force' flag to fail") + + def test_fs_new_with_specific_id_fails_already_in_use(self): + """ + That creating file system with ID already in use fails. + """ + fs_name = "test_fs_specific_id" + # file system ID already in use + fscid = self.fs.status().map['filesystems'][0]['id'] + keys = ['metadata', 'data'] + pool_names = [fs_name+'-'+key for key in keys] + for p in pool_names: + self.run_cluster_cmd(f'osd pool create {p}') + try: + self.run_cluster_cmd(f'fs new {fs_name} {pool_names[0]} {pool_names[1]} --fscid {fscid} --force') + except CommandFailedError as ce: + self.assertEqual(ce.exitstatus, errno.EINVAL, + "invalid error code on creating a file system with specifc ID that is already in use") + else: + self.fail("expected creating file system with ID already in use to fail") + + +class TestDump(CephFSTestCase): + CLIENTS_REQUIRED = 0 + MDSS_REQUIRED = 1 + + def test_fs_dump_epoch(self): + """ + That dumping a specific epoch works. + """ + + status1 = self.fs.status() + status2 = self.fs.status(epoch=status1["epoch"]-1) + self.assertEqual(status1["epoch"], status2["epoch"]+1) + + def test_fsmap_trim(self): + """ + That the fsmap is trimmed normally. + """ + + paxos_service_trim_min = 25 + self.config_set('mon', 'paxos_service_trim_min', paxos_service_trim_min) + mon_max_mdsmap_epochs = 20 + self.config_set('mon', 'mon_max_mdsmap_epochs', mon_max_mdsmap_epochs) + + status = self.fs.status() + epoch = status["epoch"] + + # for N mutations + mutations = paxos_service_trim_min + mon_max_mdsmap_epochs + b = False + for i in range(mutations): + self.fs.set_joinable(b) + b = not b + + time.sleep(10) # for tick/compaction + + try: + self.fs.status(epoch=epoch) + except CommandFailedError as e: + self.assertEqual(e.exitstatus, errno.ENOENT, "invalid error code when trying to fetch FSMap that was trimmed") + else: + self.fail("trimming did not occur as expected") + + def test_fsmap_force_trim(self): + """ + That the fsmap is trimmed forcefully. + """ + + status = self.fs.status() + epoch = status["epoch"] + + paxos_service_trim_min = 1 + self.config_set('mon', 'paxos_service_trim_min', paxos_service_trim_min) + mon_mds_force_trim_to = epoch+1 + self.config_set('mon', 'mon_mds_force_trim_to', mon_mds_force_trim_to) + + # force a new fsmap + self.fs.set_joinable(False) + time.sleep(10) # for tick/compaction + + status = self.fs.status() + log.debug(f"new epoch is {status['epoch']}") + self.fs.status(epoch=epoch+1) # epoch+1 is not trimmed, may not == status["epoch"] + + try: + self.fs.status(epoch=epoch) + except CommandFailedError as e: + self.assertEqual(e.exitstatus, errno.ENOENT, "invalid error code when trying to fetch FSMap that was trimmed") + else: + self.fail("trimming did not occur as expected") + class TestRequiredClientFeatures(CephFSTestCase): CLIENTS_REQUIRED = 0 MDSS_REQUIRED = 1 @@ -265,6 +407,141 @@ class TestRequiredClientFeatures(CephFSTestCase): p = self.fs.required_client_features('rm', '1', stderr=StringIO()) self.assertIn("removed feature 'reserved' from required_client_features", p.stderr.getvalue()) +class TestCompatCommands(CephFSTestCase): + """ + """ + + CLIENTS_REQUIRED = 0 + MDSS_REQUIRED = 3 + + def test_add_compat(self): + """ + Test adding a compat. + """ + + self.fs.fail() + self.fs.add_compat(63, 'placeholder') + mdsmap = self.fs.get_mds_map() + self.assertIn("feature_63", mdsmap['compat']['compat']) + + def test_add_incompat(self): + """ + Test adding an incompat. + """ + + self.fs.fail() + self.fs.add_incompat(63, 'placeholder') + mdsmap = self.fs.get_mds_map() + log.info(f"{mdsmap}") + self.assertIn("feature_63", mdsmap['compat']['incompat']) + + def test_rm_compat(self): + """ + Test removing a compat. + """ + + self.fs.fail() + self.fs.add_compat(63, 'placeholder') + self.fs.rm_compat(63) + mdsmap = self.fs.get_mds_map() + self.assertNotIn("feature_63", mdsmap['compat']['compat']) + + def test_rm_incompat(self): + """ + Test removing an incompat. + """ + + self.fs.fail() + self.fs.add_incompat(63, 'placeholder') + self.fs.rm_incompat(63) + mdsmap = self.fs.get_mds_map() + self.assertNotIn("feature_63", mdsmap['compat']['incompat']) + + def test_standby_compat(self): + """ + That adding a compat does not prevent standbys from joining. + """ + + self.fs.fail() + self.fs.add_compat(63, "placeholder") + self.fs.set_joinable() + self.fs.wait_for_daemons() + mdsmap = self.fs.get_mds_map() + self.assertIn("feature_63", mdsmap['compat']['compat']) + + def test_standby_incompat_reject(self): + """ + That adding an incompat feature prevents incompatible daemons from joining. + """ + + self.fs.fail() + self.fs.add_incompat(63, "placeholder") + self.fs.set_joinable() + try: + self.fs.wait_for_daemons(timeout=60) + except RuntimeError as e: + if "Timed out waiting for MDS daemons to become healthy" in str(e): + pass + else: + raise + else: + self.fail() + + def test_standby_incompat_upgrade(self): + """ + That an MDS can upgrade the compat of a fs. + """ + + self.fs.fail() + self.fs.rm_incompat(1) + self.fs.set_joinable() + self.fs.wait_for_daemons() + mdsmap = self.fs.get_mds_map() + self.assertIn("feature_1", mdsmap['compat']['incompat']) + + def test_standby_replay_not_upgradeable(self): + """ + That the mons will not upgrade the MDSMap compat if standby-replay is + enabled. + """ + + self.fs.fail() + self.fs.rm_incompat(1) + self.fs.set_allow_standby_replay(True) + self.fs.set_joinable() + try: + self.fs.wait_for_daemons(timeout=60) + except RuntimeError as e: + if "Timed out waiting for MDS daemons to become healthy" in str(e): + pass + else: + raise + else: + self.fail() + + def test_standby_incompat_reject_multifs(self): + """ + Like test_standby_incompat_reject but with a second fs. + """ + + fs2 = self.mds_cluster.newfs(name="cephfs2", create=True) + fs2.fail() + fs2.add_incompat(63, 'placeholder') + fs2.set_joinable() + try: + fs2.wait_for_daemons(timeout=60) + except RuntimeError as e: + if "Timed out waiting for MDS daemons to become healthy" in str(e): + pass + else: + raise + else: + self.fail() + # did self.fs lose MDS or standbys suicide? + self.fs.wait_for_daemons() + mdsmap = fs2.get_mds_map() + self.assertIn("feature_63", mdsmap['compat']['incompat']) + class TestConfigCommands(CephFSTestCase): """ Test that daemons and clients respond to the otherwise rarely-used diff --git a/ceph/qa/tasks/cephfs/test_cap_flush.py b/ceph/qa/tasks/cephfs/test_cap_flush.py index 2fc9410d1..c472e85bd 100644 --- a/ceph/qa/tasks/cephfs/test_cap_flush.py +++ b/ceph/qa/tasks/cephfs/test_cap_flush.py @@ -41,10 +41,10 @@ class TestCapFlush(CephFSTestCase): fd = os.open("{1}", os.O_CREAT | os.O_RDWR, 0o644) os.fchmod(fd, 0o640) """).format(dir_path, file_name) - self.mount_a.run_python(py_script) + self.mount_a.run_python(py_script, sudo=True) # Modify file mode by different user. ceph-fuse will send a setattr request - self.mount_a.run_shell(["chmod", "600", file_path], wait=False) + self.mount_a.run_shell(["chmod", "600", file_path], wait=False, sudo=True) time.sleep(10) diff --git a/ceph/qa/tasks/cephfs/test_cephfs_shell.py b/ceph/qa/tasks/cephfs/test_cephfs_shell.py index 2833c3ea2..83ee39911 100644 --- a/ceph/qa/tasks/cephfs/test_cephfs_shell.py +++ b/ceph/qa/tasks/cephfs/test_cephfs_shell.py @@ -137,7 +137,7 @@ class TestCephFSShell(CephFSTestCase): scriptfile.write(script) # copy script to the machine running cephfs-shell. mount_x.client_remote.put_file(scriptpath, scriptpath) - mount_x.run_shell('chmod 755 ' + scriptpath) + mount_x.run_shell_payload(f"chmod 755 {scriptpath}") args = ["cephfs-shell", '-b', scriptpath] if shell_conf_path: @@ -321,8 +321,7 @@ class TestGetAndPut(TestCephFSShell): size = i + 1 ofarg = 'of=' + path.join(tempdir, file_) bsarg = 'bs=' + str(size) + 'M' - self.mount_a.run_shell(['dd', 'if=/dev/urandom', ofarg, bsarg, - 'count=1']) + self.mount_a.run_shell_payload(f"dd if=/dev/urandom {ofarg} {bsarg} count=1") self.run_cephfs_shell_cmd('put ' + tempdir) for file_ in files: @@ -332,16 +331,15 @@ class TestGetAndPut(TestCephFSShell): self.mount_a.stat(path.join(self.mount_a.mountpoint, tempdirname, file_)) - self.mount_a.run_shell(['rm', '-rf', tempdir]) + self.mount_a.run_shell_payload(f"rm -rf {tempdir}") self.run_cephfs_shell_cmd('get ' + tempdirname) pwd = self.get_cephfs_shell_cmd_output('!pwd') for file_ in files: if file_ == tempdirname: - self.mount_a.run_shell('stat ' + path.join(pwd, file_)) + self.mount_a.run_shell_payload(f"stat {path.join(pwd, file_)}") else: - self.mount_a.run_shell('stat ' + path.join(pwd, tempdirname, - file_)) + self.mount_a.run_shell_payload(f"stat {path.join(pwd, tempdirname, file_)}") def test_get_with_target_name(self): """ @@ -488,7 +486,7 @@ class TestCD(TestCephFSShell): to root directory. """ path = 'dir1/dir2/dir3' - self.mount_a.run_shell('mkdir -p ' + path) + self.mount_a.run_shell_payload(f"mkdir -p {path}") expected_cwd = '/' script = 'cd {}\ncd\ncwd\n'.format(path) @@ -501,7 +499,7 @@ class TestCD(TestCephFSShell): to the path passed in the argument. """ path = 'dir1/dir2/dir3' - self.mount_a.run_shell('mkdir -p ' + path) + self.mount_a.run_shell_payload(f"mkdir -p {path}") expected_cwd = '/dir1/dir2/dir3' script = 'cd {}\ncwd\n'.format(path) @@ -514,8 +512,7 @@ class TestDU(TestCephFSShell): def test_du_works_for_regfiles(self): regfilename = 'some_regfile' regfile_abspath = path.join(self.mount_a.mountpoint, regfilename) - self.mount_a.client_remote.write_file(regfile_abspath, - 'somedata', sudo=True) + self.mount_a.client_remote.write_file(regfile_abspath, 'somedata') size = humansize(self.mount_a.stat(regfile_abspath)['st_size']) expected_output = r'{}{}{}'.format(size, " +", regfilename) @@ -528,9 +525,8 @@ class TestDU(TestCephFSShell): dir_abspath = path.join(self.mount_a.mountpoint, dirname) regfilename = 'some_regfile' regfile_abspath = path.join(dir_abspath, regfilename) - self.mount_a.run_shell('mkdir ' + dir_abspath) - self.mount_a.client_remote.write_file(regfile_abspath, - 'somedata', sudo=True) + self.mount_a.run_shell_payload(f"mkdir {dir_abspath}") + self.mount_a.client_remote.write_file(regfile_abspath, 'somedata') # XXX: we stat `regfile_abspath` here because ceph du reports a non-empty # directory's size as sum of sizes of all files under it. @@ -544,7 +540,7 @@ class TestDU(TestCephFSShell): def test_du_works_for_empty_dirs(self): dirname = 'some_directory' dir_abspath = path.join(self.mount_a.mountpoint, dirname) - self.mount_a.run_shell('mkdir ' + dir_abspath) + self.mount_a.run_shell_payload(f"mkdir {dir_abspath}") size = humansize(self.mount_a.stat(dir_abspath)['st_size']) expected_output = r'{}{}{}'.format(size, " +", dirname) @@ -555,12 +551,10 @@ class TestDU(TestCephFSShell): def test_du_works_for_hardlinks(self): regfilename = 'some_regfile' regfile_abspath = path.join(self.mount_a.mountpoint, regfilename) - self.mount_a.client_remote.write_file(regfile_abspath, - 'somedata', sudo=True) + self.mount_a.client_remote.write_file(regfile_abspath, 'somedata') hlinkname = 'some_hardlink' hlink_abspath = path.join(self.mount_a.mountpoint, hlinkname) - self.mount_a.run_shell(['sudo', 'ln', regfile_abspath, - hlink_abspath], omit_sudo=False) + self.mount_a.run_shell_payload(f"ln {regfile_abspath} {hlink_abspath}") size = humansize(self.mount_a.stat(hlink_abspath)['st_size']) expected_output = r'{}{}{}'.format(size, " +", hlinkname) @@ -571,11 +565,10 @@ class TestDU(TestCephFSShell): def test_du_works_for_softlinks_to_files(self): regfilename = 'some_regfile' regfile_abspath = path.join(self.mount_a.mountpoint, regfilename) - self.mount_a.client_remote.write_file(regfile_abspath, - 'somedata', sudo=True) + self.mount_a.client_remote.write_file(regfile_abspath, 'somedata') slinkname = 'some_softlink' slink_abspath = path.join(self.mount_a.mountpoint, slinkname) - self.mount_a.run_shell(['ln', '-s', regfile_abspath, slink_abspath]) + self.mount_a.run_shell_payload(f"ln -s {regfile_abspath} {slink_abspath}") size = humansize(self.mount_a.lstat(slink_abspath)['st_size']) expected_output = r'{}{}{}'.format((size), " +", slinkname) @@ -586,10 +579,10 @@ class TestDU(TestCephFSShell): def test_du_works_for_softlinks_to_dirs(self): dirname = 'some_directory' dir_abspath = path.join(self.mount_a.mountpoint, dirname) - self.mount_a.run_shell('mkdir ' + dir_abspath) + self.mount_a.run_shell_payload(f"mkdir {dir_abspath}") slinkname = 'some_softlink' slink_abspath = path.join(self.mount_a.mountpoint, slinkname) - self.mount_a.run_shell(['ln', '-s', dir_abspath, slink_abspath]) + self.mount_a.run_shell_payload(f"ln -s {dir_abspath} {slink_abspath}") size = humansize(self.mount_a.lstat(slink_abspath)['st_size']) expected_output = r'{}{}{}'.format(size, " +", slinkname) @@ -612,11 +605,11 @@ class TestDU(TestCephFSShell): slink_abspath = path.join(self.mount_a.mountpoint, slinkname) slink2_abspath = path.join(self.mount_a.mountpoint, slink2name) - self.mount_a.run_shell('mkdir ' + dir_abspath) - self.mount_a.run_shell('touch ' + regfile_abspath) - self.mount_a.run_shell(['ln', regfile_abspath, hlink_abspath]) - self.mount_a.run_shell(['ln', '-s', regfile_abspath, slink_abspath]) - self.mount_a.run_shell(['ln', '-s', dir_abspath, slink2_abspath]) + self.mount_a.run_shell_payload(f"mkdir {dir_abspath}") + self.mount_a.run_shell_payload(f"touch {regfile_abspath}") + self.mount_a.run_shell_payload(f"ln {regfile_abspath} {hlink_abspath}") + self.mount_a.run_shell_payload(f"ln -s {regfile_abspath} {slink_abspath}") + self.mount_a.run_shell_payload(f"ln -s {dir_abspath} {slink2_abspath}") dir2_name = 'dir2' dir21_name = 'dir21' @@ -624,13 +617,11 @@ class TestDU(TestCephFSShell): dir2_abspath = path.join(self.mount_a.mountpoint, dir2_name) dir21_abspath = path.join(dir2_abspath, dir21_name) regfile121_abspath = path.join(dir21_abspath, regfile121_name) - self.mount_a.run_shell('mkdir -p ' + dir21_abspath) - self.mount_a.run_shell('touch ' + regfile121_abspath) + self.mount_a.run_shell_payload(f"mkdir -p {dir21_abspath}") + self.mount_a.run_shell_payload(f"touch {regfile121_abspath}") - self.mount_a.client_remote.write_file(regfile_abspath, - 'somedata', sudo=True) - self.mount_a.client_remote.write_file(regfile121_abspath, - 'somemoredata', sudo=True) + self.mount_a.client_remote.write_file(regfile_abspath, 'somedata') + self.mount_a.client_remote.write_file(regfile121_abspath, 'somemoredata') # TODO: is there a way to trigger/force update ceph.dir.rbytes? # wait so that attr ceph.dir.rbytes gets a chance to be updated. @@ -731,7 +722,7 @@ class TestDF(TestCephFSShell): def test_df_for_valid_directory(self): dir_name = 'dir1' - mount_output = self.mount_a.run_shell('mkdir ' + dir_name) + mount_output = self.mount_a.run_shell_payload(f"mkdir {dir_name}") log.info("cephfs-shell mount output:\n{}".format(mount_output)) self.validate_df(dir_name) @@ -799,10 +790,10 @@ class TestQuota(TestCephFSShell): def test_exceed_file_limit(self): self.test_set() dir_abspath = path.join(self.mount_a.mountpoint, self.dir_name) - self.mount_a.run_shell('touch '+dir_abspath+'/file1') + self.mount_a.run_shell_payload(f"touch {dir_abspath}/file1") file2 = path.join(dir_abspath, "file2") try: - self.mount_a.run_shell('touch '+file2) + self.mount_a.run_shell_payload(f"touch {file2}") raise Exception("Something went wrong!! File creation should have failed") except CommandFailedError: # Test should pass as file quota set to 2 @@ -818,8 +809,7 @@ class TestQuota(TestCephFSShell): file_abspath = path.join(dir_abspath, filename) try: # Write should fail as bytes quota is set to 6 - self.mount_a.client_remote.write_file(file_abspath, - 'Disk raise Exception', sudo=True) + self.mount_a.client_remote.write_file(file_abspath, 'Disk raise Exception') raise Exception("Write should have failed") except CommandFailedError: # Test should pass only when write command fails @@ -924,8 +914,8 @@ class TestLS(TestCephFSShell): for (file_size, file_name) in zip(file_sizes, file_names): temp_file = self.mount_a.client_remote.mktemp(file_name) - self.mount_a.run_shell(f"fallocate -l {file_size} {temp_file}") - self.mount_a.run_shell(f'mv {temp_file} ./') + self.mount_a.run_shell_payload(f"fallocate -l {file_size} {temp_file}") + self.mount_a.run_shell_payload(f'mv {temp_file} ./') ls_H_output = self.get_cephfs_shell_cmd_output(['ls', '-lH']) diff --git a/ceph/qa/tasks/cephfs/test_client_recovery.py b/ceph/qa/tasks/cephfs/test_client_recovery.py index 3ae208a69..24726b369 100644 --- a/ceph/qa/tasks/cephfs/test_client_recovery.py +++ b/ceph/qa/tasks/cephfs/test_client_recovery.py @@ -135,9 +135,11 @@ class TestClientRecovery(CephFSTestCase): # ================= # Check that if I stop an MDS and a client goes away, the MDS waits # for the reconnect period - self.fs.fail() mount_a_client_id = self.mount_a.get_global_id() + + self.fs.fail() + self.mount_a.umount_wait(force=True) self.fs.set_joinable() @@ -508,9 +510,8 @@ class TestClientRecovery(CephFSTestCase): self.assertEqual(current_readdirs, initial_readdirs); mount_b_gid = self.mount_b.get_global_id() - mount_b_pid = self.mount_b.get_client_pid() # stop ceph-fuse process of mount_b - self.mount_b.client_remote.run(args=["sudo", "kill", "-STOP", mount_b_pid]) + self.mount_b.suspend_netns() self.assert_session_state(mount_b_gid, "open") time.sleep(session_timeout * 1.5) # Long enough for MDS to consider session stale @@ -519,7 +520,7 @@ class TestClientRecovery(CephFSTestCase): self.assert_session_state(mount_b_gid, "stale") # resume ceph-fuse process of mount_b - self.mount_b.client_remote.run(args=["sudo", "kill", "-CONT", mount_b_pid]) + self.mount_b.resume_netns() # Is the new file visible from mount_b? (caps become invalid after session stale) self.mount_b.run_shell(["ls", "testdir/file2"]) @@ -615,10 +616,10 @@ class TestClientRecovery(CephFSTestCase): self.mount_a.umount_wait() if isinstance(self.mount_a, FuseMount): - self.mount_a.mount(mntopts=['--client_reconnect_stale=1', '--fuse_disable_pagecache=1']) + self.mount_a.mount_wait(mntopts=['--client_reconnect_stale=1', '--fuse_disable_pagecache=1']) else: try: - self.mount_a.mount(mntopts=['recover_session=clean']) + self.mount_a.mount_wait(mntopts=['recover_session=clean']) except CommandFailedError: self.mount_a.kill_cleanup() self.skipTest("Not implemented in current kernel") @@ -682,7 +683,7 @@ class TestClientRecovery(CephFSTestCase): raise RuntimeError("read() failed to raise error") """).format(path=path) rproc = self.mount_a.client_remote.run( - args=['sudo', 'python3', '-c', pyscript], + args=['python3', '-c', pyscript], wait=False, stdin=run.PIPE, stdout=run.PIPE) rproc.stdout.readline() diff --git a/ceph/qa/tasks/cephfs/test_data_scan.py b/ceph/qa/tasks/cephfs/test_data_scan.py index 933a7f67d..dcb7eda40 100644 --- a/ceph/qa/tasks/cephfs/test_data_scan.py +++ b/ceph/qa/tasks/cephfs/test_data_scan.py @@ -82,8 +82,8 @@ class SimpleWorkload(Workload): self._initial_state = self._mount.stat("subdir/sixmegs") def validate(self): - self._mount.run_shell(["ls", "subdir"]) - st = self._mount.stat("subdir/sixmegs") + self._mount.run_shell(["ls", "subdir"], sudo=True) + st = self._mount.stat("subdir/sixmegs", sudo=True) self.assert_equal(st['st_size'], self._initial_state['st_size']) return self._errors @@ -104,8 +104,8 @@ class MovedFile(Workload): pass def validate(self): - self.assert_equal(self._mount.ls(), ["subdir_alpha"]) - st = self._mount.stat("subdir_alpha/sixmegs") + self.assert_equal(self._mount.ls(sudo=True), ["subdir_alpha"]) + st = self._mount.stat("subdir_alpha/sixmegs", sudo=True) self.assert_equal(st['st_size'], self._initial_state['st_size']) return self._errors @@ -124,9 +124,9 @@ class BacktracelessFile(Workload): ino_name = "%x" % self._initial_state["st_ino"] # The inode should be linked into lost+found because we had no path for it - self.assert_equal(self._mount.ls(), ["lost+found"]) - self.assert_equal(self._mount.ls("lost+found"), [ino_name]) - st = self._mount.stat("lost+found/{ino_name}".format(ino_name=ino_name)) + self.assert_equal(self._mount.ls(sudo=True), ["lost+found"]) + self.assert_equal(self._mount.ls("lost+found", sudo=True), [ino_name]) + st = self._mount.stat(f"lost+found/{ino_name}", sudo=True) # We might not have got the name or path, but we should still get the size self.assert_equal(st['st_size'], self._initial_state['st_size']) @@ -200,7 +200,7 @@ class StripedStashedLayout(Workload): # The unflushed file should have been recovered into lost+found without # the correct layout: read back junk ino_name = "%x" % self._initial_state["unflushed_ino"] - self.assert_equal(self._mount.ls("lost+found"), [ino_name]) + self.assert_equal(self._mount.ls("lost+found", sudo=True), [ino_name]) try: self._mount.validate_test_pattern(os.path.join("lost+found", ino_name), 1024 * 512) except CommandFailedError: @@ -259,8 +259,8 @@ class MovedDir(Workload): self.assert_equal(len(root_files), 1) self.assert_equal(root_files[0] in ["grandfather", "grandmother"], True) winner = root_files[0] - st_opf = self._mount.stat("{0}/parent/orig_pos_file".format(winner)) - st_npf = self._mount.stat("{0}/parent/new_pos_file".format(winner)) + st_opf = self._mount.stat(f"{winner}/parent/orig_pos_file", sudo=True) + st_npf = self._mount.stat(f"{winner}/parent/new_pos_file", sudo=True) self.assert_equal(st_opf['st_size'], self._initial_state[0]['st_size']) self.assert_equal(st_npf['st_size'], self._initial_state[1]['st_size']) @@ -278,7 +278,8 @@ class MissingZerothObject(Workload): self._filesystem.rados(["rm", zeroth_id], pool=self._filesystem.get_data_pool_name()) def validate(self): - st = self._mount.stat("lost+found/{0:x}".format(self._initial_state['st_ino'])) + ino = self._initial_state['st_ino'] + st = self._mount.stat(f"lost+found/{ino:x}", sudo=True) self.assert_equal(st['st_size'], self._initial_state['st_size']) @@ -295,12 +296,11 @@ class NonDefaultLayout(Workload): def validate(self): # Check we got the layout reconstructed properly - object_size = int(self._mount.getfattr( - "./datafile", "ceph.file.layout.object_size")) + object_size = int(self._mount.getfattr("./datafile", "ceph.file.layout.object_size", sudo=True)) self.assert_equal(object_size, 8388608) # Check we got the file size reconstructed properly - st = self._mount.stat("datafile") + st = self._mount.stat("datafile", sudo=True) self.assert_equal(st['st_size'], self._initial_state['st_size']) @@ -490,7 +490,9 @@ class TestDataScan(CephFSTestCase): self.fs.set_joinable() self.fs.wait_for_daemons() self.mount_a.mount_wait() - out = self.mount_a.run_shell(["cat", "subdir/{0}".format(victim_dentry)]).stdout.getvalue().strip() + self.mount_a.run_shell(["ls", "-l", "subdir/"]) # debugging + # Use sudo because cephfs-data-scan will reinsert the dentry with root ownership, it can't know the real owner. + out = self.mount_a.run_shell_payload(f"cat subdir/{victim_dentry}", sudo=True).stdout.getvalue().strip() self.assertEqual(out, victim_dentry) # Finally, close the loop by checking our injected dentry survives a merge @@ -543,7 +545,7 @@ class TestDataScan(CephFSTestCase): pgs_to_files[pgid].append(file_path) log.info("{0}: {1}".format(file_path, pgid)) - pg_count = self.fs.pgs_per_fs_pool + pg_count = self.fs.get_pool_pg_num(self.fs.get_data_pool_name()) for pg_n in range(0, pg_count): pg_str = "{0}.{1:x}".format(self.fs.get_data_pool_id(), pg_n) out = self.fs.data_scan(["pg_files", "mydir", pg_str]) diff --git a/ceph/qa/tasks/cephfs/test_failover.py b/ceph/qa/tasks/cephfs/test_failover.py index 45e343dcd..304d27c2c 100644 --- a/ceph/qa/tasks/cephfs/test_failover.py +++ b/ceph/qa/tasks/cephfs/test_failover.py @@ -650,14 +650,14 @@ class TestMultiFilesystems(CephFSTestCase): fs_a, fs_b = self._setup_two() # Mount a client on fs_a - self.mount_a.mount(cephfs_name=fs_a.name) + self.mount_a.mount_wait(cephfs_name=fs_a.name) self.mount_a.write_n_mb("pad.bin", 1) self.mount_a.write_n_mb("test.bin", 2) a_created_ino = self.mount_a.path_to_ino("test.bin") self.mount_a.create_files() # Mount a client on fs_b - self.mount_b.mount(cephfs_name=fs_b.name) + self.mount_b.mount_wait(cephfs_name=fs_b.name) self.mount_b.write_n_mb("test.bin", 1) b_created_ino = self.mount_b.path_to_ino("test.bin") self.mount_b.create_files() diff --git a/ceph/qa/tasks/cephfs/test_fragment.py b/ceph/qa/tasks/cephfs/test_fragment.py index 6e2823b4a..41977ca20 100644 --- a/ceph/qa/tasks/cephfs/test_fragment.py +++ b/ceph/qa/tasks/cephfs/test_fragment.py @@ -297,7 +297,7 @@ class TestFragmentation(CephFSTestCase): self.mount_a.run_shell(["ln", "testdir1/{0}".format(i), "testdir2/"]) self.mount_a.umount_wait() - self.mount_a.mount() + self.mount_a.mount_wait() self.mount_a.wait_until_mounted() # flush journal and restart mds. after restart, testdir2 is not in mds' cache diff --git a/ceph/qa/tasks/cephfs/test_full.py b/ceph/qa/tasks/cephfs/test_full.py index 541525486..2b7166d6b 100644 --- a/ceph/qa/tasks/cephfs/test_full.py +++ b/ceph/qa/tasks/cephfs/test_full.py @@ -25,9 +25,8 @@ class FullnessTestCase(CephFSTestCase): pool_capacity = None # type: Optional[int] fill_mb = None - # Subclasses define what fullness means to them def is_full(self): - raise NotImplementedError() + return self.fs.is_full() def setUp(self): CephFSTestCase.setUp(self) @@ -126,14 +125,14 @@ class FullnessTestCase(CephFSTestCase): # how soon the cluster recognises its own fullness self.mount_a.write_n_mb("large_file_a", self.fill_mb // 2) try: - self.mount_a.write_n_mb("large_file_b", self.fill_mb // 2) + self.mount_a.write_n_mb("large_file_b", (self.fill_mb * 1.1) // 2) except CommandFailedError: log.info("Writing file B failed (full status happened already)") assert self.is_full() else: log.info("Writing file B succeeded (full status will happen soon)") self.wait_until_true(lambda: self.is_full(), - timeout=osd_mon_report_interval * 5) + timeout=osd_mon_report_interval * 120) # Attempting to write more data should give me ENOSPC with self.assertRaises(CommandFailedError) as ar: @@ -168,7 +167,7 @@ class FullnessTestCase(CephFSTestCase): # * The MDS to purge the stray folder and execute object deletions # * The OSDs to inform the mon that they are no longer full self.wait_until_true(lambda: not self.is_full(), - timeout=osd_mon_report_interval * 5) + timeout=osd_mon_report_interval * 120) # Wait for the MDS to see the latest OSD map so that it will reliably # be applying the free space policy @@ -214,9 +213,13 @@ class FullnessTestCase(CephFSTestCase): log.warning("This test may run rather slowly unless you decrease" "osd_mon_report_interval (5 is a good setting)!") + # set the object_size to 1MB to make the objects destributed more evenly + # among the OSDs to fix Tracker#45434 + file_layout = "stripe_unit=1048576 stripe_count=1 object_size=1048576" self.mount_a.run_python(template.format( fill_mb=self.fill_mb, file_path=file_path, + file_layout=file_layout, full_wait=full_wait, is_fuse=isinstance(self.mount_a, FuseMount) )) @@ -234,6 +237,7 @@ class FullnessTestCase(CephFSTestCase): print("writing some data through which we expect to succeed") bytes = 0 f = os.open("{file_path}", os.O_WRONLY | os.O_CREAT) + os.setxattr("{file_path}", 'ceph.file.layout', b'{file_layout}') bytes += os.write(f, b'a' * 512 * 1024) os.fsync(f) print("fsync'ed data successfully, will now attempt to fill fs") @@ -305,6 +309,7 @@ class FullnessTestCase(CephFSTestCase): print("writing some data through which we expect to succeed") bytes = 0 f = os.open("{file_path}", os.O_WRONLY | os.O_CREAT) + os.setxattr("{file_path}", 'ceph.file.layout', b'{file_layout}') bytes += os.write(f, b'a' * 4096) os.fsync(f) print("fsync'ed data successfully, will now attempt to fill fs") @@ -320,8 +325,13 @@ class FullnessTestCase(CephFSTestCase): bytes += os.write(f, b'x' * 1024 * 1024) print("wrote bytes via buffered write, moving on to fsync") except OSError as e: - print("Unexpected error %s from write() instead of fsync()" % e) - raise + if {is_fuse}: + print("Unexpected error %s from write() instead of fsync()" % e) + raise + else: + print("Reached fullness after %.2f MB" % (bytes / (1024.0 * 1024.0))) + full = True + break try: os.fsync(f) @@ -373,9 +383,6 @@ class TestQuotaFull(FullnessTestCase): self.fs.mon_manager.raw_cluster_cmd("osd", "pool", "set-quota", pool_name, "max_bytes", "{0}".format(self.pool_capacity)) - def is_full(self): - return self.fs.is_full() - class TestClusterFull(FullnessTestCase): """ @@ -388,13 +395,8 @@ class TestClusterFull(FullnessTestCase): super(TestClusterFull, self).setUp() if self.pool_capacity is None: - max_avail = self.fs.get_pool_df(self._data_pool_name())['max_avail'] - full_ratio = float(self.fs.get_config("mon_osd_full_ratio", service_type="mon")) - TestClusterFull.pool_capacity = int(max_avail * full_ratio) + TestClusterFull.pool_capacity = self.fs.get_pool_df(self._data_pool_name())['max_avail'] TestClusterFull.fill_mb = (self.pool_capacity // (1024 * 1024)) - def is_full(self): - return self.fs.is_full() - # Hide the parent class so that unittest.loader doesn't try to run it. del globals()['FullnessTestCase'] diff --git a/ceph/qa/tasks/cephfs/test_mirroring.py b/ceph/qa/tasks/cephfs/test_mirroring.py index 6ea3d5d30..c3c746a1d 100644 --- a/ceph/qa/tasks/cephfs/test_mirroring.py +++ b/ceph/qa/tasks/cephfs/test_mirroring.py @@ -480,7 +480,7 @@ class TestMirroring(CephFSTestCase): log.debug(f'mounting filesystem {self.secondary_fs_name}') self.mount_b.umount_wait() - self.mount_b.mount(cephfs_name=self.secondary_fs_name) + self.mount_b.mount_wait(cephfs_name=self.secondary_fs_name) # create a bunch of files in a directory to snap self.mount_a.run_shell(["mkdir", "d0"]) @@ -546,7 +546,7 @@ class TestMirroring(CephFSTestCase): log.debug(f'mounting filesystem {self.secondary_fs_name}') self.mount_b.umount_wait() - self.mount_b.mount(cephfs_name=self.secondary_fs_name) + self.mount_b.mount_wait(cephfs_name=self.secondary_fs_name) # create a bunch of files in a directory to snap self.mount_a.run_shell(["mkdir", "d0"]) @@ -582,7 +582,7 @@ class TestMirroring(CephFSTestCase): log.debug(f'mounting filesystem {self.secondary_fs_name}') self.mount_b.umount_wait() - self.mount_b.mount(cephfs_name=self.secondary_fs_name) + self.mount_b.mount_wait(cephfs_name=self.secondary_fs_name) # create a bunch of files in a directory to snap self.mount_a.run_shell(["mkdir", "d0"]) @@ -818,7 +818,7 @@ class TestMirroring(CephFSTestCase): log.debug(f'mounting filesystem {self.secondary_fs_name}') self.mount_b.umount_wait() - self.mount_b.mount(cephfs_name=self.secondary_fs_name) + self.mount_b.mount_wait(cephfs_name=self.secondary_fs_name) # create a bunch of files w/ symbolic links in a directory to snap self.mount_a.run_shell(["mkdir", "d0"]) @@ -955,7 +955,7 @@ class TestMirroring(CephFSTestCase): self.backup_fs.get_data_pool_name(), self.backup_fs.get_data_pool_name())) log.debug(f'mounting filesystem {self.secondary_fs_name}') self.mount_b.umount_wait() - self.mount_b.mount(cephfs_name=self.secondary_fs_name) + self.mount_b.mount_wait(cephfs_name=self.secondary_fs_name) repo = 'ceph-qa-suite' repo_dir = 'ceph_repo' @@ -1033,7 +1033,7 @@ class TestMirroring(CephFSTestCase): self.backup_fs.get_data_pool_name(), self.backup_fs.get_data_pool_name())) log.debug(f'mounting filesystem {self.secondary_fs_name}') self.mount_b.umount_wait() - self.mount_b.mount(cephfs_name=self.secondary_fs_name) + self.mount_b.mount_wait(cephfs_name=self.secondary_fs_name) typs = deque(['reg', 'dir', 'sym']) def cleanup_and_create_with_type(dirname, fnames): @@ -1104,7 +1104,7 @@ class TestMirroring(CephFSTestCase): self.backup_fs.get_data_pool_name(), self.backup_fs.get_data_pool_name())) log.debug(f'mounting filesystem {self.secondary_fs_name}') self.mount_b.umount_wait() - self.mount_b.mount(cephfs_name=self.secondary_fs_name) + self.mount_b.mount_wait(cephfs_name=self.secondary_fs_name) repo = 'ceph-qa-suite' repo_dir = 'ceph_repo' @@ -1166,3 +1166,105 @@ class TestMirroring(CephFSTestCase): raise RuntimeError('adding peer should fail') self.disable_mirroring(self.primary_fs_name, self.primary_fs_id) + + def test_cephfs_mirror_cancel_mirroring_and_readd(self): + """ + Test adding a directory path for synchronization post removal of already added directory paths + + ... to ensure that synchronization of the newly added directory path functions + as expected. Note that we schedule three (3) directories for mirroring to ensure + that all replayer threads (3 by default) in the mirror daemon are busy. + """ + log.debug('reconfigure client auth caps') + self.mds_cluster.mon_manager.raw_cluster_cmd_result( + 'auth', 'caps', "client.{0}".format(self.mount_b.client_id), + 'mds', 'allow rw', + 'mon', 'allow r', + 'osd', 'allow rw pool={0}, allow rw pool={1}'.format( + self.backup_fs.get_data_pool_name(), self.backup_fs.get_data_pool_name())) + + log.debug(f'mounting filesystem {self.secondary_fs_name}') + self.mount_b.umount_wait() + self.mount_b.mount_wait(cephfs_name=self.secondary_fs_name) + + # create a bunch of files in a directory to snap + self.mount_a.run_shell(["mkdir", "d0"]) + self.mount_a.run_shell(["mkdir", "d1"]) + self.mount_a.run_shell(["mkdir", "d2"]) + for i in range(4): + filename = f'file.{i}' + self.mount_a.write_n_mb(os.path.join('d0', filename), 1024) + self.mount_a.write_n_mb(os.path.join('d1', filename), 1024) + self.mount_a.write_n_mb(os.path.join('d2', filename), 1024) + + log.debug('enabling mirroring') + self.enable_mirroring(self.primary_fs_name, self.primary_fs_id) + log.debug('adding directory paths') + self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d0') + self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d1') + self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d2') + self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name) + + # take snapshots + log.debug('taking snapshots') + self.mount_a.run_shell(["mkdir", "d0/.snap/snap0"]) + self.mount_a.run_shell(["mkdir", "d1/.snap/snap0"]) + self.mount_a.run_shell(["mkdir", "d2/.snap/snap0"]) + + time.sleep(10) + log.debug('checking snap in progress') + self.check_peer_snap_in_progress(self.primary_fs_name, self.primary_fs_id, + "client.mirror_remote@ceph", '/d0', 'snap0') + self.check_peer_snap_in_progress(self.primary_fs_name, self.primary_fs_id, + "client.mirror_remote@ceph", '/d1', 'snap0') + self.check_peer_snap_in_progress(self.primary_fs_name, self.primary_fs_id, + "client.mirror_remote@ceph", '/d2', 'snap0') + + log.debug('removing directories 1') + self.remove_directory(self.primary_fs_name, self.primary_fs_id, '/d0') + log.debug('removing directories 2') + self.remove_directory(self.primary_fs_name, self.primary_fs_id, '/d1') + log.debug('removing directories 3') + self.remove_directory(self.primary_fs_name, self.primary_fs_id, '/d2') + + log.debug('removing snapshots') + self.mount_a.run_shell(["rmdir", "d0/.snap/snap0"]) + self.mount_a.run_shell(["rmdir", "d1/.snap/snap0"]) + self.mount_a.run_shell(["rmdir", "d2/.snap/snap0"]) + + for i in range(4): + filename = f'file.{i}' + log.debug(f'deleting {filename}') + self.mount_a.run_shell(["rm", "-f", os.path.join('d0', filename)]) + self.mount_a.run_shell(["rm", "-f", os.path.join('d1', filename)]) + self.mount_a.run_shell(["rm", "-f", os.path.join('d2', filename)]) + + log.debug('creating new files...') + self.mount_a.create_n_files('d0/file', 50, sync=True) + self.mount_a.create_n_files('d1/file', 50, sync=True) + self.mount_a.create_n_files('d2/file', 50, sync=True) + + log.debug('adding directory paths') + self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d0') + self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d1') + self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d2') + + log.debug('creating new snapshots...') + self.mount_a.run_shell(["mkdir", "d0/.snap/snap0"]) + self.mount_a.run_shell(["mkdir", "d1/.snap/snap0"]) + self.mount_a.run_shell(["mkdir", "d2/.snap/snap0"]) + + time.sleep(60) + self.check_peer_status(self.primary_fs_name, self.primary_fs_id, + "client.mirror_remote@ceph", '/d0', 'snap0', 1) + self.verify_snapshot('d0', 'snap0') + + self.check_peer_status(self.primary_fs_name, self.primary_fs_id, + "client.mirror_remote@ceph", '/d1', 'snap0', 1) + self.verify_snapshot('d1', 'snap0') + + self.check_peer_status(self.primary_fs_name, self.primary_fs_id, + "client.mirror_remote@ceph", '/d2', 'snap0', 1) + self.verify_snapshot('d2', 'snap0') + + self.disable_mirroring(self.primary_fs_name, self.primary_fs_id) diff --git a/ceph/qa/tasks/cephfs/test_misc.py b/ceph/qa/tasks/cephfs/test_misc.py index cf3b1a1bf..a6cb3da4e 100644 --- a/ceph/qa/tasks/cephfs/test_misc.py +++ b/ceph/qa/tasks/cephfs/test_misc.py @@ -82,7 +82,7 @@ class TestMisc(CephFSTestCase): '--yes-i-really-really-mean-it') self.fs.mon_manager.raw_cluster_cmd('osd', 'pool', 'create', self.fs.metadata_pool_name, - self.fs.pgs_per_fs_pool.__str__()) + '--pg_num_min', str(self.fs.pg_num_min)) # insert a garbage object self.fs.radosm(["put", "foo", "-"], stdin=StringIO("bar")) @@ -119,7 +119,7 @@ class TestMisc(CephFSTestCase): '--yes-i-really-really-mean-it') self.fs.mon_manager.raw_cluster_cmd('osd', 'pool', 'create', self.fs.metadata_pool_name, - self.fs.pgs_per_fs_pool.__str__()) + '--pg_num_min', str(self.fs.pg_num_min)) self.fs.mon_manager.raw_cluster_cmd('fs', 'new', self.fs.name, self.fs.metadata_pool_name, data_pool_name) diff --git a/ceph/qa/tasks/cephfs/test_nfs.py b/ceph/qa/tasks/cephfs/test_nfs.py index 83c4797b2..c3feb1604 100644 --- a/ceph/qa/tasks/cephfs/test_nfs.py +++ b/ceph/qa/tasks/cephfs/test_nfs.py @@ -264,9 +264,11 @@ class TestNFS(MgrTestCase): return raise + self.ctx.cluster.run(args=['sudo', 'chmod', '1777', '/mnt']) + try: - self.ctx.cluster.run(args=['sudo', 'touch', '/mnt/test']) - out_mnt = self._sys_cmd(['sudo', 'ls', '/mnt']) + self.ctx.cluster.run(args=['touch', '/mnt/test']) + out_mnt = self._sys_cmd(['ls', '/mnt']) self.assertEqual(out_mnt, b'test\n') finally: self.ctx.cluster.run(args=['sudo', 'umount', '/mnt']) diff --git a/ceph/qa/tasks/cephfs/test_scrub_checks.py b/ceph/qa/tasks/cephfs/test_scrub_checks.py index f1af60480..bcfc2fc9a 100644 --- a/ceph/qa/tasks/cephfs/test_scrub_checks.py +++ b/ceph/qa/tasks/cephfs/test_scrub_checks.py @@ -303,8 +303,8 @@ class TestScrubChecks(CephFSTestCase): mds_rank = 0 test_dir = "scrub_repair_path" - self.mount_a.run_shell(["sudo", "mkdir", test_dir]) - self.mount_a.run_shell(["sudo", "touch", "{0}/file".format(test_dir)]) + self.mount_a.run_shell(["mkdir", test_dir]) + self.mount_a.run_shell(["touch", "{0}/file".format(test_dir)]) dir_objname = "{:x}.00000000".format(self.mount_a.path_to_ino(test_dir)) self.mount_a.umount_wait() @@ -323,7 +323,7 @@ class TestScrubChecks(CephFSTestCase): # fragstat indicates the directory is not empty, rmdir should fail with self.assertRaises(CommandFailedError) as ar: - self.mount_a.run_shell(["sudo", "rmdir", test_dir]) + self.mount_a.run_shell(["rmdir", test_dir]) self.assertEqual(ar.exception.exitstatus, 1) self.tell_command(mds_rank, "scrub start /{0} repair".format(test_dir), @@ -333,7 +333,7 @@ class TestScrubChecks(CephFSTestCase): time.sleep(10) # fragstat should be fixed - self.mount_a.run_shell(["sudo", "rmdir", test_dir]) + self.mount_a.run_shell(["rmdir", test_dir]) @staticmethod def json_validator(json_out, rc, element, expected_value): diff --git a/ceph/qa/tasks/cephfs/test_sessionmap.py b/ceph/qa/tasks/cephfs/test_sessionmap.py index 79f1fb45e..ad6fd1d60 100644 --- a/ceph/qa/tasks/cephfs/test_sessionmap.py +++ b/ceph/qa/tasks/cephfs/test_sessionmap.py @@ -41,21 +41,26 @@ class TestSessionMap(CephFSTestCase): the conn count goes back to where it started (i.e. we aren't leaving connections open) """ + self.config_set('mds', 'ms_async_reap_threshold', '1') + self.mount_a.umount_wait() self.mount_b.umount_wait() status = self.fs.status() s = self._get_connection_count(status=status) self.fs.rank_tell(["session", "ls"], status=status) - e = self._get_connection_count(status=status) - - self.assertEqual(s, e) + self.wait_until_true( + lambda: self._get_connection_count(status=status) == s, + timeout=30 + ) def test_mount_conn_close(self): """ That when a client unmounts, the thread count on the MDS goes back to what it was before the client mounted """ + self.config_set('mds', 'ms_async_reap_threshold', '1') + self.mount_a.umount_wait() self.mount_b.umount_wait() @@ -64,9 +69,10 @@ class TestSessionMap(CephFSTestCase): self.mount_a.mount_wait() self.assertGreater(self._get_connection_count(status=status), s) self.mount_a.umount_wait() - e = self._get_connection_count(status=status) - - self.assertEqual(s, e) + self.wait_until_true( + lambda: self._get_connection_count(status=status) == s, + timeout=30 + ) def test_version_splitting(self): """ diff --git a/ceph/qa/tasks/cephfs/test_volumes.py b/ceph/qa/tasks/cephfs/test_volumes.py index a759d55ec..f5dc9fa55 100644 --- a/ceph/qa/tasks/cephfs/test_volumes.py +++ b/ceph/qa/tasks/cephfs/test_volumes.py @@ -233,20 +233,20 @@ class TestVolumesHelper(CephFSTestCase): subvolpath = self._get_subvolume_path(self.volname, subvolume, group_name=subvolume_group) if pool is not None: - self.mount_a.setfattr(subvolpath, 'ceph.dir.layout.pool', pool) + self.mount_a.setfattr(subvolpath, 'ceph.dir.layout.pool', pool, sudo=True) if pool_namespace is not None: - self.mount_a.setfattr(subvolpath, 'ceph.dir.layout.pool_namespace', pool_namespace) + self.mount_a.setfattr(subvolpath, 'ceph.dir.layout.pool_namespace', pool_namespace, sudo=True) def _do_subvolume_attr_update(self, subvolume, uid, gid, mode, subvolume_group=None): subvolpath = self._get_subvolume_path(self.volname, subvolume, group_name=subvolume_group) # mode - self.mount_a.run_shell(['chmod', mode, subvolpath]) + self.mount_a.run_shell(['chmod', mode, subvolpath], sudo=True) # ownership - self.mount_a.run_shell(['chown', uid, subvolpath]) - self.mount_a.run_shell(['chgrp', gid, subvolpath]) + self.mount_a.run_shell(['chown', uid, subvolpath], sudo=True) + self.mount_a.run_shell(['chgrp', gid, subvolpath], sudo=True) def _do_subvolume_io(self, subvolume, subvolume_group=None, create_dir=None, number_of_files=DEFAULT_NUMBER_OF_FILES, file_size=DEFAULT_FILE_SIZE): @@ -262,7 +262,7 @@ class TestVolumesHelper(CephFSTestCase): io_path = subvolpath if create_dir: io_path = os.path.join(subvolpath, create_dir) - self.mount_a.run_shell(["mkdir", "-p", io_path]) + self.mount_a.run_shell_payload(f"mkdir -p {io_path}") log.debug("filling subvolume {0} with {1} files each {2}MB size under directory {3}".format(subvolume, number_of_files, file_size, io_path)) for i in range(number_of_files): @@ -278,11 +278,11 @@ class TestVolumesHelper(CephFSTestCase): # this symlink's ownership would be changed sym_path2 = os.path.join(dir_path, "sym.0") - self.mount_a.run_shell(["sudo", "mkdir", dir_path], omit_sudo=False) - self.mount_a.run_shell(["sudo", "ln", "-s", "./{}".format(reg_file), sym_path1], omit_sudo=False) - self.mount_a.run_shell(["sudo", "ln", "-s", "./{}".format(reg_file), sym_path2], omit_sudo=False) + self.mount_a.run_shell(["mkdir", dir_path]) + self.mount_a.run_shell(["ln", "-s", "./{}".format(reg_file), sym_path1]) + self.mount_a.run_shell(["ln", "-s", "./{}".format(reg_file), sym_path2]) # flip ownership to nobody. assumption: nobody's id is 65534 - self.mount_a.run_shell(["sudo", "chown", "-h", "65534:65534", sym_path2], omit_sudo=False) + self.mount_a.run_shell(["chown", "-h", "65534:65534", sym_path2], sudo=True, omit_sudo=False) def _wait_for_trash_empty(self, timeout=30): # XXX: construct the trash dir path (note that there is no mgr @@ -301,7 +301,7 @@ class TestVolumesHelper(CephFSTestCase): group = subvol_group if subvol_group is not None else '_nogroup' metapath = os.path.join(".", "volumes", group, subvol_name, ".meta") - out = self.mount_a.run_shell(['cat', metapath]) + out = self.mount_a.run_shell(['cat', metapath], sudo=True) lines = out.stdout.getvalue().strip().split('\n') sv_version = -1 for line in lines: @@ -316,16 +316,16 @@ class TestVolumesHelper(CephFSTestCase): basepath = os.path.join("volumes", group, subvol_name) uuid_str = str(uuid.uuid4()) createpath = os.path.join(basepath, uuid_str) - self.mount_a.run_shell(['mkdir', '-p', createpath]) + self.mount_a.run_shell(['mkdir', '-p', createpath], sudo=True) # create a v1 snapshot, to prevent auto upgrades if has_snapshot: snappath = os.path.join(createpath, ".snap", "fake") - self.mount_a.run_shell(['mkdir', '-p', snappath]) + self.mount_a.run_shell(['mkdir', '-p', snappath], sudo=True) # add required xattrs to subvolume default_pool = self.mount_a.getfattr(".", "ceph.dir.layout.pool") - self.mount_a.setfattr(createpath, 'ceph.dir.layout.pool', default_pool) + self.mount_a.setfattr(createpath, 'ceph.dir.layout.pool', default_pool, sudo=True) # create a v1 .meta file meta_contents = "[GLOBAL]\nversion = 1\ntype = {0}\npath = {1}\nstate = {2}\n".format(subvol_type, "/" + createpath, state) @@ -333,17 +333,16 @@ class TestVolumesHelper(CephFSTestCase): # add a fake clone source meta_contents = meta_contents + '[source]\nvolume = fake\nsubvolume = fake\nsnapshot = fake\n' meta_filepath1 = os.path.join(self.mount_a.mountpoint, basepath, ".meta") - self.mount_a.client_remote.write_file(meta_filepath1, - meta_contents, sudo=True) + self.mount_a.client_remote.write_file(meta_filepath1, meta_contents, sudo=True) return createpath def _update_fake_trash(self, subvol_name, subvol_group=None, trash_name='fake', create=True): group = subvol_group if subvol_group is not None else '_nogroup' trashpath = os.path.join("volumes", group, subvol_name, '.trash', trash_name) if create: - self.mount_a.run_shell(['mkdir', '-p', trashpath]) + self.mount_a.run_shell(['mkdir', '-p', trashpath], sudo=True) else: - self.mount_a.run_shell(['rmdir', trashpath]) + self.mount_a.run_shell(['rmdir', trashpath], sudo=True) def _configure_guest_auth(self, guest_mount, authid, key): """ @@ -612,7 +611,7 @@ class TestSubvolumeGroups(TestVolumesHelper): # create group self._fs_cmd("subvolumegroup", "create", self.volname, group1) - self._fs_cmd("subvolumegroup", "create", self.volname, group2, "--mode", "777") + self._fs_cmd("subvolumegroup", "create", self.volname, group2, f"--mode={expected_mode2}") group1_path = self._get_subvolume_group_path(self.volname, group1) group2_path = self._get_subvolume_group_path(self.volname, group2) @@ -726,7 +725,7 @@ class TestSubvolumes(TestVolumesHelper): # create subvolumes for subvolume in subvolumes: - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") self._do_subvolume_io(subvolume, number_of_files=10) self.mount_a.umount_wait() @@ -1157,7 +1156,7 @@ class TestSubvolumes(TestVolumesHelper): else: raise RuntimeError("expected renaming subvolume incarnation out of subvolume directory to fail") """) - self.mount_a.run_python(rename_script.format(src=srcpath, dst=dstpath)) + self.mount_a.run_python(rename_script.format(src=srcpath, dst=dstpath), sudo=True) # remove subvolume self._fs_cmd("subvolume", "rm", self.volname, subvolume) @@ -1196,11 +1195,11 @@ class TestSubvolumes(TestVolumesHelper): # emulate a old-fashioned subvolume in a custom group createpath = os.path.join(".", "volumes", group, subvolume) - self.mount_a.run_shell(['mkdir', '-p', createpath]) + self.mount_a.run_shell(['mkdir', '-p', createpath], sudo=True) # add required xattrs to subvolume default_pool = self.mount_a.getfattr(".", "ceph.dir.layout.pool") - self.mount_a.setfattr(createpath, 'ceph.dir.layout.pool', default_pool) + self.mount_a.setfattr(createpath, 'ceph.dir.layout.pool', default_pool, sudo=True) mount_path = os.path.join("/", "volumes", group, subvolume) @@ -1216,7 +1215,7 @@ class TestSubvolumes(TestVolumesHelper): self._configure_guest_auth(guest_mount, authid, key) # mount the subvolume, and write to it - guest_mount.mount(cephfs_mntpt=mount_path) + guest_mount.mount_wait(cephfs_mntpt=mount_path) guest_mount.write_n_mb("data.bin", 1) # authorize guest authID read access to subvolume @@ -1226,7 +1225,7 @@ class TestSubvolumes(TestVolumesHelper): # guest client sees the change in access level to read only after a # remount of the subvolume. guest_mount.umount_wait() - guest_mount.mount(cephfs_mntpt=mount_path) + guest_mount.mount_wait(cephfs_mntpt=mount_path) # read existing content of the subvolume self.assertListEqual(guest_mount.ls(guest_mount.mountpoint), ["data.bin"]) @@ -1253,7 +1252,7 @@ class TestSubvolumes(TestVolumesHelper): guest_mount.umount_wait() # create group - self._fs_cmd("subvolumegroup", "create", self.volname, group) + self._fs_cmd("subvolumegroup", "create", self.volname, group, "--mode=777") # create subvolume in group self._fs_cmd("subvolume", "create", self.volname, subvolume, "--group_name", group) @@ -1272,7 +1271,7 @@ class TestSubvolumes(TestVolumesHelper): self._configure_guest_auth(guest_mount, authid, key) # mount the subvolume, and write to it - guest_mount.mount(cephfs_mntpt=mount_path) + guest_mount.mount_wait(cephfs_mntpt=mount_path) guest_mount.write_n_mb("data.bin", 1) # authorize guest authID read access to subvolume @@ -1282,7 +1281,7 @@ class TestSubvolumes(TestVolumesHelper): # guest client sees the change in access level to read only after a # remount of the subvolume. guest_mount.umount_wait() - guest_mount.mount(cephfs_mntpt=mount_path) + guest_mount.mount_wait(cephfs_mntpt=mount_path) # read existing content of the subvolume self.assertListEqual(guest_mount.ls(guest_mount.mountpoint), ["data.bin"]) @@ -1630,7 +1629,7 @@ class TestSubvolumes(TestVolumesHelper): # Induce partial auth update state by modifying the auth metadata file, # and then run authorize again. - guest_mount.run_shell(['sed', '-i', 's/false/true/g', 'volumes/{0}'.format(auth_metadata_filename)]) + guest_mount.run_shell(['sed', '-i', 's/false/true/g', 'volumes/{0}'.format(auth_metadata_filename)], sudo=True) # Authorize 'guestclient_1' to access the subvolume. self._fs_cmd("subvolume", "authorize", self.volname, subvolume, guestclient_1["auth_id"], @@ -1686,7 +1685,7 @@ class TestSubvolumes(TestVolumesHelper): # Induce partial auth update state by modifying the auth metadata file, # and then run de-authorize. - guest_mount.run_shell(['sed', '-i', 's/false/true/g', 'volumes/{0}'.format(auth_metadata_filename)]) + guest_mount.run_shell(['sed', '-i', 's/false/true/g', 'volumes/{0}'.format(auth_metadata_filename)], sudo=True) # Deauthorize 'guestclient_1' to access the subvolume2. self._fs_cmd("subvolume", "deauthorize", self.volname, subvolume2, guestclient_1["auth_id"], @@ -1739,7 +1738,7 @@ class TestSubvolumes(TestVolumesHelper): self.assertIn(auth_metadata_filename, guest_mount.ls("volumes")) # Replace 'subvolumes' to 'volumes', old style auth-metadata file - guest_mount.run_shell(['sed', '-i', 's/subvolumes/volumes/g', 'volumes/{0}'.format(auth_metadata_filename)]) + guest_mount.run_shell(['sed', '-i', 's/subvolumes/volumes/g', 'volumes/{0}'.format(auth_metadata_filename)], sudo=True) # Authorize 'guestclient_1' to access the subvolume2. This should transparently update 'volumes' to 'subvolumes' self._fs_cmd("subvolume", "authorize", self.volname, subvolume2, guestclient_1["auth_id"], @@ -1817,7 +1816,7 @@ class TestSubvolumes(TestVolumesHelper): self.assertIn(auth_metadata_filename, guest_mount.ls("volumes")) # Replace 'subvolumes' to 'volumes', old style auth-metadata file - guest_mount.run_shell(['sed', '-i', 's/subvolumes/volumes/g', 'volumes/{0}'.format(auth_metadata_filename)]) + guest_mount.run_shell(['sed', '-i', 's/subvolumes/volumes/g', 'volumes/{0}'.format(auth_metadata_filename)], sudo=True) # Deauthorize 'guestclient_1' to access the subvolume2. This should update 'volumes' to subvolumes' self._fs_cmd("subvolume", "deauthorize", self.volname, subvolume2, auth_id, "--group_name", group) @@ -1875,7 +1874,7 @@ class TestSubvolumes(TestVolumesHelper): # subvolumes. Mount the two subvolumes. Write data to the volumes. for i in range(2): # Create subvolume. - self._fs_cmd("subvolume", "create", self.volname, subvolumes[i], "--group_name", group) + self._fs_cmd("subvolume", "create", self.volname, subvolumes[i], "--group_name", group, "--mode=777") # authorize guest authID read-write access to subvolume key = self._fs_cmd("subvolume", "authorize", self.volname, subvolumes[i], guestclient_1["auth_id"], @@ -1887,7 +1886,7 @@ class TestSubvolumes(TestVolumesHelper): self._configure_guest_auth(guest_mounts[i], auth_id, key) # mount the subvolume, and write to it - guest_mounts[i].mount(cephfs_mntpt=mount_path) + guest_mounts[i].mount_wait(cephfs_mntpt=mount_path) guest_mounts[i].write_n_mb("data.bin", 1) # Evict client, guest_mounts[0], using auth ID 'guest' and has mounted @@ -2010,7 +2009,7 @@ class TestSubvolumes(TestVolumesHelper): osize = self.DEFAULT_FILE_SIZE*1024*1024*20 # create subvolume subvolname = self._generate_random_subvolume_name() - self._fs_cmd("subvolume", "create", self.volname, subvolname, "--size", str(osize)) + self._fs_cmd("subvolume", "create", self.volname, subvolname, "--size", str(osize), "--mode=777") # make sure it exists subvolpath = self._get_subvolume_path(self.volname, subvolname) @@ -2057,7 +2056,7 @@ class TestSubvolumes(TestVolumesHelper): osize = self.DEFAULT_FILE_SIZE*1024*1024*20 # create subvolume subvolname = self._generate_random_subvolume_name() - self._fs_cmd("subvolume", "create", self.volname, subvolname, "--size", str(osize)) + self._fs_cmd("subvolume", "create", self.volname, subvolname, "--size", str(osize), "--mode=777") # make sure it exists subvolpath = self._get_subvolume_path(self.volname, subvolname) @@ -2105,7 +2104,7 @@ class TestSubvolumes(TestVolumesHelper): osize = self.DEFAULT_FILE_SIZE*1024*1024*10 # create subvolume of quota 10MB and make sure it exists subvolname = self._generate_random_subvolume_name() - self._fs_cmd("subvolume", "create", self.volname, subvolname, "--size", str(osize)) + self._fs_cmd("subvolume", "create", self.volname, subvolname, "--size", str(osize), "--mode=777") subvolpath = self._get_subvolume_path(self.volname, subvolname) self.assertNotEqual(subvolpath, None) @@ -2181,7 +2180,7 @@ class TestSubvolumes(TestVolumesHelper): # create subvolume subvolname = self._generate_random_subvolume_name() self._fs_cmd("subvolume", "create", self.volname, subvolname, "--size", - str(self.DEFAULT_FILE_SIZE*1024*1024*5)) + str(self.DEFAULT_FILE_SIZE*1024*1024*5), "--mode=777") # make sure it exists subvolpath = self._get_subvolume_path(self.volname, subvolname) @@ -2482,7 +2481,7 @@ class TestSubvolumeSnapshots(TestVolumesHelper): snapshot, snap_missing = self._generate_random_snapshot_name(2) # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # do some IO self._do_subvolume_io(subvolume, number_of_files=1) @@ -2594,13 +2593,13 @@ class TestSubvolumeSnapshots(TestVolumesHelper): # Create snapshot at ancestral level ancestral_snappath1 = os.path.join(".", "volumes", group, ".snap", "ancestral_snap_1") ancestral_snappath2 = os.path.join(".", "volumes", group, ".snap", "ancestral_snap_2") - self.mount_a.run_shell(['mkdir', '-p', ancestral_snappath1, ancestral_snappath2]) + self.mount_a.run_shell(['mkdir', '-p', ancestral_snappath1, ancestral_snappath2], sudo=True) subvolsnapshotls = json.loads(self._fs_cmd('subvolume', 'snapshot', 'ls', self.volname, subvolume, group)) self.assertEqual(len(subvolsnapshotls), snap_count) # remove ancestral snapshots - self.mount_a.run_shell(['rmdir', ancestral_snappath1, ancestral_snappath2]) + self.mount_a.run_shell(['rmdir', ancestral_snappath1, ancestral_snappath2], sudo=True) # remove snapshot for snapshot in snapshots: @@ -2634,7 +2633,7 @@ class TestSubvolumeSnapshots(TestVolumesHelper): # Create snapshot at ancestral level ancestral_snap_name = "ancestral_snap_1" ancestral_snappath1 = os.path.join(".", "volumes", group, ".snap", ancestral_snap_name) - self.mount_a.run_shell(['mkdir', '-p', ancestral_snappath1]) + self.mount_a.run_shell(['mkdir', '-p', ancestral_snappath1], sudo=True) # Validate existence of inherited snapshot group_path = os.path.join(".", "volumes", group) @@ -2652,7 +2651,7 @@ class TestSubvolumeSnapshots(TestVolumesHelper): self.fail("expected snapshot info of inherited snapshot to fail") # remove ancestral snapshots - self.mount_a.run_shell(['rmdir', ancestral_snappath1]) + self.mount_a.run_shell(['rmdir', ancestral_snappath1], sudo=True) # remove subvolume self._fs_cmd("subvolume", "rm", self.volname, subvolume, "--group_name", group) @@ -2682,7 +2681,7 @@ class TestSubvolumeSnapshots(TestVolumesHelper): # Create snapshot at ancestral level ancestral_snap_name = "ancestral_snap_1" ancestral_snappath1 = os.path.join(".", "volumes", group, ".snap", ancestral_snap_name) - self.mount_a.run_shell(['mkdir', '-p', ancestral_snappath1]) + self.mount_a.run_shell(['mkdir', '-p', ancestral_snappath1], sudo=True) # Validate existence of inherited snap group_path = os.path.join(".", "volumes", group) @@ -2700,7 +2699,7 @@ class TestSubvolumeSnapshots(TestVolumesHelper): self.fail("expected removing inheirted snapshot to fail") # remove ancestral snapshots - self.mount_a.run_shell(['rmdir', ancestral_snappath1]) + self.mount_a.run_shell(['rmdir', ancestral_snappath1], sudo=True) # remove subvolume self._fs_cmd("subvolume", "rm", self.volname, subvolume, group) @@ -2730,7 +2729,7 @@ class TestSubvolumeSnapshots(TestVolumesHelper): # Create subvolumegroup snapshot group_snapshot_path = os.path.join(".", "volumes", group, ".snap", group_snapshot) - self.mount_a.run_shell(['mkdir', '-p', group_snapshot_path]) + self.mount_a.run_shell(['mkdir', '-p', group_snapshot_path], sudo=True) # Validate existence of subvolumegroup snapshot self.mount_a.run_shell(['ls', group_snapshot_path]) @@ -2744,7 +2743,7 @@ class TestSubvolumeSnapshots(TestVolumesHelper): self.fail("expected subvolume snapshot creation with same name as subvolumegroup snapshot to fail") # remove subvolumegroup snapshot - self.mount_a.run_shell(['rmdir', group_snapshot_path]) + self.mount_a.run_shell(['rmdir', group_snapshot_path], sudo=True) # remove subvolume self._fs_cmd("subvolume", "rm", self.volname, subvolume, group) @@ -3057,7 +3056,7 @@ class TestSubvolumeSnapshots(TestVolumesHelper): clone = self._generate_random_clone_name() # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # do some IO self._do_subvolume_io(subvolume, number_of_files=64) @@ -3115,7 +3114,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): clone = self._generate_random_clone_name() # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # do some IO self._do_subvolume_io(subvolume, number_of_files=1) @@ -3175,7 +3174,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): osize = self.DEFAULT_FILE_SIZE*1024*1024*12 # create subvolume, in an isolated namespace with a specified size - self._fs_cmd("subvolume", "create", self.volname, subvolume, "--namespace-isolated", "--size", str(osize)) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--namespace-isolated", "--size", str(osize), "--mode=777") # do some IO self._do_subvolume_io(subvolume, number_of_files=8) @@ -3218,7 +3217,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): clone = self._generate_random_clone_name() # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # do some IO self._do_subvolume_io(subvolume, number_of_files=64) @@ -3226,6 +3225,9 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) + # Insert delay at the beginning of snapshot clone + self.config_set('mgr', 'mgr/volumes/snapshot_clone_delay', 2) + # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) @@ -3264,7 +3266,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): clone = self._generate_random_clone_name() # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # do some IO self._do_subvolume_io(subvolume, number_of_files=64) @@ -3272,6 +3274,9 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) + # Insert delay at the beginning of snapshot clone + self.config_set('mgr', 'mgr/volumes/snapshot_clone_delay', 2) + # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) @@ -3309,7 +3314,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): clone = self._generate_random_clone_name() # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # do some IO self._do_subvolume_io(subvolume, number_of_files=64) @@ -3317,6 +3322,9 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) + # Insert delay at the beginning of snapshot clone + self.config_set('mgr', 'mgr/volumes/snapshot_clone_delay', 2) + # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) @@ -3357,7 +3365,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): clone = self._generate_random_clone_name() # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # store path for clone verification subvol1_path = self._get_subvolume_path(self.volname, subvolume) @@ -3431,7 +3439,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): clone = self._generate_random_clone_name() # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # store path for clone verification subvol_path = self._get_subvolume_path(self.volname, subvolume) @@ -3476,7 +3484,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): clone = self._generate_random_clone_name(1) # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # do some IO self._do_subvolume_io(subvolume, number_of_files=16) @@ -3488,7 +3496,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): self._fs_cmd("subvolume", "rm", self.volname, subvolume, "--retain-snapshots") # recreate subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # get and store path for clone verification subvol2_path = self._get_subvolume_path(self.volname, subvolume) @@ -3533,7 +3541,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): snapshot = self._generate_random_snapshot_name() # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # store path for clone verification subvol_path = self._get_subvolume_path(self.volname, subvolume) @@ -3632,7 +3640,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): clone = self._generate_random_clone_name() # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # do some IO self._do_subvolume_io_mixed(subvolume) @@ -3665,7 +3673,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): clone = self._generate_random_clone_name() # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # do some IO self._do_subvolume_io(subvolume, number_of_files=64) @@ -3698,7 +3706,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): clone = self._generate_random_clone_name() # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # Create a file with suid, guid bits set along with executable bit. args = ["subvolume", "getpath", self.volname, subvolume] @@ -3740,7 +3748,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): clone1, clone2 = self._generate_random_clone_name(2) # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # do some IO self._do_subvolume_io(subvolume, number_of_files=32) @@ -3793,7 +3801,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): clone = self._generate_random_clone_name() # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # do some IO self._do_subvolume_io(subvolume, number_of_files=128) @@ -3801,6 +3809,9 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): # snapshot subvolume self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) + # Insert delay at the beginning of snapshot clone + self.config_set('mgr', 'mgr/volumes/snapshot_clone_delay', 2) + # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) @@ -3841,7 +3852,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): clones = self._generate_random_clone_name(NR_CLONES) # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # do some IO self._do_subvolume_io(subvolume, number_of_files=4, file_size=FILE_SIZE_MB) @@ -3898,7 +3909,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): self._fs_cmd("subvolumegroup", "create", self.volname, c_group) # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume, s_group) + self._fs_cmd("subvolume", "create", self.volname, subvolume, s_group, "--mode=777") # do some IO self._do_subvolume_io(subvolume, subvolume_group=s_group, number_of_files=32) @@ -3940,7 +3951,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): nr_files = int((pool_capacity * 0.99) / (TestVolumes.DEFAULT_FILE_SIZE * 1024 * 1024)) # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # do some IO self._do_subvolume_io(subvolume, number_of_files=nr_files) @@ -3999,8 +4010,8 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): clone = self._generate_random_clone_name() # create subvolumes - self._fs_cmd("subvolume", "create", self.volname, subvolume1) - self._fs_cmd("subvolume", "create", self.volname, subvolume2) + self._fs_cmd("subvolume", "create", self.volname, subvolume1, "--mode=777") + self._fs_cmd("subvolume", "create", self.volname, subvolume2, "--mode=777") # do some IO self._do_subvolume_io(subvolume1, number_of_files=32) @@ -4055,7 +4066,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): newid = self.fs.add_data_pool(new_pool) # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # do some IO self._do_subvolume_io(subvolume, number_of_files=32) @@ -4096,7 +4107,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): group = self._generate_random_group_name() # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume) + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--mode=777") # do some IO self._do_subvolume_io(subvolume, number_of_files=32) @@ -4185,11 +4196,11 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): # emulate a old-fashioned subvolume createpath = os.path.join(".", "volumes", "_nogroup", subvolume) - self.mount_a.run_shell(['mkdir', '-p', createpath]) + self.mount_a.run_shell_payload(f"mkdir -p -m 777 {createpath}", sudo=True) # add required xattrs to subvolume default_pool = self.mount_a.getfattr(".", "ceph.dir.layout.pool") - self.mount_a.setfattr(createpath, 'ceph.dir.layout.pool', default_pool) + self.mount_a.setfattr(createpath, 'ceph.dir.layout.pool', default_pool, sudo=True) # do some IO self._do_subvolume_io(subvolume, number_of_files=64) @@ -4200,6 +4211,9 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): # ensure metadata file is in legacy location, with required version v1 self._assert_meta_location_and_version(self.volname, subvolume, version=1, legacy=True) + # Insert delay at the beginning of snapshot clone + self.config_set('mgr', 'mgr/volumes/snapshot_clone_delay', 2) + # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) @@ -4249,6 +4263,25 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): max_concurrent_clones = int(self.config_get('mgr', 'mgr/volumes/max_concurrent_clones')) self.assertEqual(max_concurrent_clones, 2) + def test_subvolume_snapshot_config_snapshot_clone_delay(self): + """ + Validate 'snapshot_clone_delay' config option + """ + + # get the default delay before starting the clone + default_timeout = int(self.config_get('mgr', 'mgr/volumes/snapshot_clone_delay')) + self.assertEqual(default_timeout, 0) + + # Insert delay of 2 seconds at the beginning of the snapshot clone + self.config_set('mgr', 'mgr/volumes/snapshot_clone_delay', 2) + default_timeout = int(self.config_get('mgr', 'mgr/volumes/snapshot_clone_delay')) + self.assertEqual(default_timeout, 2) + + # Decrease number of cloner threads + self.config_set('mgr', 'mgr/volumes/max_concurrent_clones', 2) + max_concurrent_clones = int(self.config_get('mgr', 'mgr/volumes/max_concurrent_clones')) + self.assertEqual(max_concurrent_clones, 2) + def test_subvolume_under_group_snapshot_clone(self): subvolume = self._generate_random_subvolume_name() group = self._generate_random_group_name() @@ -4259,7 +4292,7 @@ class TestSubvolumeSnapshotClones(TestVolumesHelper): self._fs_cmd("subvolumegroup", "create", self.volname, group) # create subvolume - self._fs_cmd("subvolume", "create", self.volname, subvolume, group) + self._fs_cmd("subvolume", "create", self.volname, subvolume, group, "--mode=777") # do some IO self._do_subvolume_io(subvolume, subvolume_group=group, number_of_files=32) @@ -4426,11 +4459,11 @@ class TestMisc(TestVolumesHelper): # emulate a old-fashioned subvolume -- one in the default group and # the other in a custom group createpath1 = os.path.join(".", "volumes", "_nogroup", subvolume1) - self.mount_a.run_shell(['mkdir', '-p', createpath1]) + self.mount_a.run_shell(['mkdir', '-p', createpath1], sudo=True) # create group createpath2 = os.path.join(".", "volumes", group, subvolume2) - self.mount_a.run_shell(['mkdir', '-p', createpath2]) + self.mount_a.run_shell(['mkdir', '-p', createpath2], sudo=True) # this would auto-upgrade on access without anyone noticing subvolpath1 = self._fs_cmd("subvolume", "getpath", self.volname, subvolume1) diff --git a/ceph/qa/tasks/mds_pre_upgrade.py b/ceph/qa/tasks/mds_pre_upgrade.py index 0856d4833..812d402ed 100644 --- a/ceph/qa/tasks/mds_pre_upgrade.py +++ b/ceph/qa/tasks/mds_pre_upgrade.py @@ -3,7 +3,6 @@ Prepare MDS cluster for upgrade. """ import logging -import time from tasks.cephfs.filesystem import Filesystem @@ -22,22 +21,7 @@ def task(ctx, config): 'snap-upgrade task only accepts a dict for configuration' fs = Filesystem(ctx) - status = fs.getinfo() - + fs.getinfo() # load name + fs.set_allow_standby_replay(False) fs.set_max_mds(1) fs.reach_max_mds() - - # Stop standbys now to minimize time rank 0 is down in subsequent: - # tasks: - # - ceph.stop: [mds.*] - rank0 = fs.get_rank(rank=0, status=status) - for daemon in ctx.daemons.iter_daemons_of_role('mds', fs.mon_manager.cluster): - if rank0['name'] != daemon.id_: - daemon.stop() - - for i in range(1, 10): - time.sleep(5) # time for FSMap to update - status = fs.getinfo() - if len(list(status.get_standbys())) == 0: - break - assert(len(list(status.get_standbys())) == 0) diff --git a/ceph/qa/tasks/mgr/dashboard/test_motd.py b/ceph/qa/tasks/mgr/dashboard/test_motd.py new file mode 100644 index 000000000..2edbf36ba --- /dev/null +++ b/ceph/qa/tasks/mgr/dashboard/test_motd.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# pylint: disable=too-many-public-methods + +from __future__ import absolute_import + +import time + +from .helper import DashboardTestCase + + +class MotdTest(DashboardTestCase): + @classmethod + def tearDownClass(cls): + cls._ceph_cmd(['dashboard', 'motd', 'clear']) + super(MotdTest, cls).tearDownClass() + + def setUp(self): + super(MotdTest, self).setUp() + self._ceph_cmd(['dashboard', 'motd', 'clear']) + + def test_none(self): + data = self._get('/ui-api/motd') + self.assertStatus(200) + self.assertIsNone(data) + + def test_set(self): + self._ceph_cmd(['dashboard', 'motd', 'set', 'info', '0', 'foo bar baz']) + data = self._get('/ui-api/motd') + self.assertStatus(200) + self.assertIsInstance(data, dict) + + def test_expired(self): + self._ceph_cmd(['dashboard', 'motd', 'set', 'info', '2s', 'foo bar baz']) + time.sleep(5) + data = self._get('/ui-api/motd') + self.assertStatus(200) + self.assertIsNone(data) diff --git a/ceph/qa/tasks/mgr/dashboard/test_rgw.py b/ceph/qa/tasks/mgr/dashboard/test_rgw.py index f545c7483..1bfb99506 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_rgw.py +++ b/ceph/qa/tasks/mgr/dashboard/test_rgw.py @@ -70,29 +70,20 @@ class RgwApiCredentialsTest(RgwTestCase): AUTH_ROLES = ['rgw-manager'] - def setUp(self): - super(RgwApiCredentialsTest, self).setUp() - # Restart the Dashboard module to ensure that the connection to the - # RGW Admin Ops API is re-established with the new credentials. - self.logout() - self._ceph_cmd(['mgr', 'module', 'disable', 'dashboard']) - self._ceph_cmd(['mgr', 'module', 'enable', 'dashboard', '--force']) - # Set the default credentials. - self._ceph_cmd_with_secret(['dashboard', 'set-rgw-api-secret-key'], 'admin') - self._ceph_cmd_with_secret(['dashboard', 'set-rgw-api-access-key'], 'admin') - super(RgwApiCredentialsTest, self).setUp() - - def test_no_access_secret_key(self): - self._ceph_cmd(['dashboard', 'reset-rgw-api-secret-key']) - self._ceph_cmd(['dashboard', 'reset-rgw-api-access-key']) + def test_invalid_credentials(self): + self._ceph_cmd_with_secret(['dashboard', 'set-rgw-api-secret-key'], 'invalid') + self._ceph_cmd_with_secret(['dashboard', 'set-rgw-api-access-key'], 'invalid') resp = self._get('/api/rgw/user') - self.assertStatus(500) + self.assertStatus(404) self.assertIn('detail', resp) self.assertIn('component', resp) - self.assertIn('No RGW credentials found', resp['detail']) + self.assertIn('Error connecting to Object Gateway', resp['detail']) self.assertEqual(resp['component'], 'rgw') def test_success(self): + # Set the default credentials. + self._ceph_cmd_with_secret(['dashboard', 'set-rgw-api-secret-key'], 'admin') + self._ceph_cmd_with_secret(['dashboard', 'set-rgw-api-access-key'], 'admin') data = self._get('/api/rgw/status') self.assertStatus(200) self.assertIn('available', data) @@ -211,6 +202,11 @@ class RgwBucketTest(RgwTestCase): 'tenant': JLeaf(str), }, allow_unknown=True)) + # List all buckets names without stats. + data = self._get('/api/rgw/bucket?stats=false') + self.assertStatus(200) + self.assertEqual(data, ['teuth-test-bucket']) + # Get the bucket. data = self._get('/api/rgw/bucket/teuth-test-bucket') self.assertStatus(200) diff --git a/ceph/qa/tasks/mgr/dashboard/test_settings.py b/ceph/qa/tasks/mgr/dashboard/test_settings.py index 46750292e..d6ad1e762 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_settings.py +++ b/ceph/qa/tasks/mgr/dashboard/test_settings.py @@ -51,15 +51,15 @@ class SettingsTest(DashboardTestCase): def test_bulk_set(self): self._put('/api/settings', { - 'RGW_API_HOST': 'somehost', - 'RGW_API_PORT': 7777, + 'RGW_API_ACCESS_KEY': 'dummy-key', + 'RGW_API_SECRET_KEY': 'dummy-secret', }) self.assertStatus(200) - host = self._get('/api/settings/rgw-api-host')['value'] + access_key = self._get('/api/settings/rgw-api-access-key')['value'] self.assertStatus(200) - self.assertEqual('somehost', host) + self.assertEqual('dummy-key', access_key) - port = self._get('/api/settings/rgw-api-port')['value'] + secret_key = self._get('/api/settings/rgw-api-secret-key')['value'] self.assertStatus(200) - self.assertEqual(7777, port) + self.assertEqual('dummy-secret', secret_key) diff --git a/ceph/qa/tasks/mgr/test_insights.py b/ceph/qa/tasks/mgr/test_insights.py index 9f952152c..aa2548881 100644 --- a/ceph/qa/tasks/mgr/test_insights.py +++ b/ceph/qa/tasks/mgr/test_insights.py @@ -149,17 +149,6 @@ class TestInsights(MgrTestCase): active_id = self.mgr_cluster.get_active_id() self.mgr_cluster.mgr_restart(active_id) - # ensure that at least one of the checks is present after the restart. - # we don't for them all to be present because "earlier" checks may not - # have sat in memory long enough to be flushed. - all_missing = True - report = self._insights() - for check in check_names: - if check in report["health"]["history"]["checks"]: - all_missing = False - break - self.assertFalse(all_missing) - # pruning really removes history self.mgr_cluster.mon_manager.raw_cluster_cmd_result( "insights", "prune-health", "0") diff --git a/ceph/qa/workunits/cephadm/test_cephadm.sh b/ceph/qa/workunits/cephadm/test_cephadm.sh index 2bc94b88c..10ccd2458 100755 --- a/ceph/qa/workunits/cephadm/test_cephadm.sh +++ b/ceph/qa/workunits/cephadm/test_cephadm.sh @@ -161,6 +161,9 @@ systemctl status docker > /dev/null && ( $CEPHADM --docker version | grep 'ceph $CEPHADM shell --fsid $FSID -- ceph -v | grep 'ceph version' $CEPHADM shell --fsid $FSID -e FOO=BAR -- printenv | grep FOO=BAR +# test stdin +echo foo | $CEPHADM shell -- cat | grep -q foo + ## bootstrap ORIG_CONFIG=`mktemp -p $TMPDIR` CONFIG=`mktemp -p $TMPDIR` diff --git a/ceph/qa/workunits/mon/pg_autoscaler.sh b/ceph/qa/workunits/mon/pg_autoscaler.sh index 706f87d0e..3d24b1a6c 100755 --- a/ceph/qa/workunits/mon/pg_autoscaler.sh +++ b/ceph/qa/workunits/mon/pg_autoscaler.sh @@ -30,6 +30,8 @@ function wait_for() { return 0 } +function power2() { echo "x=l($1)/l(2); scale=0; 2^((x+0.5)/1)" | bc -l;} + # enable ceph config set mgr mgr/pg_autoscaler/sleep_interval 5 ceph mgr module enable pg_autoscaler @@ -40,8 +42,20 @@ ceph osd pool create b 16 --pg-num-min 2 ceph osd pool set a pg_autoscale_mode on ceph osd pool set b pg_autoscale_mode on -wait_for 120 "ceph osd pool get a pg_num | grep 4" -wait_for 120 "ceph osd pool get b pg_num | grep 2" +# get num pools again since we created more pools +NUM_POOLS=$(ceph osd pool ls | wc -l) + +# get pool size +POOL_SIZE_A=$(ceph osd pool get a size| grep -Eo '[0-9]{1,4}') +POOL_SIZE_B=$(ceph osd pool get b size| grep -Eo '[0-9]{1,4}') + +# calculate target pg of each pools +TARGET_PG_A=$(power2 $((($NUM_OSDS * 100)/($NUM_POOLS)/($POOL_SIZE_A)))) +TARGET_PG_B=$(power2 $((($NUM_OSDS * 100)/($NUM_POOLS)/($POOL_SIZE_B)))) + +# evaluate target_pg against pg num of each pools +wait_for 120 "ceph osd pool get a pg_num | grep $TARGET_PG_A" +wait_for 120 "ceph osd pool get b pg_num | grep $TARGET_PG_B" # target ratio ceph osd pool set a target_size_ratio 5 diff --git a/ceph/qa/workunits/mon/test_mon_config_key.py b/ceph/qa/workunits/mon/test_mon_config_key.py index bcc1a79dd..0d8ec1c27 100755 --- a/ceph/qa/workunits/mon/test_mon_config_key.py +++ b/ceph/qa/workunits/mon/test_mon_config_key.py @@ -78,39 +78,19 @@ def run_cmd(cmd, expects=0): cmdlog = LOG.getChild('run_cmd') cmdlog.debug('{fc}'.format(fc=' '.join(full_cmd))) - proc = subprocess.Popen(full_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - stdout = [] - stderr = [] - while True: - try: - out, err = proc.communicate() - if out is not None: - stdout += out.decode().split('\n') - cmdlog.debug('stdout: {s}'.format(s=out)) - if err is not None: - stdout += err.decode().split('\n') - cmdlog.debug('stderr: {s}'.format(s=err)) - except ValueError: - ret = proc.wait() - break - - if ret != expects: - cmdlog.error('cmd > {cmd}'.format(cmd=full_cmd)) - cmdlog.error("expected return '{expected}' got '{got}'".format( - expected=expects, got=ret)) + proc = subprocess.run(full_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + if proc.returncode != expects: + cmdlog.error(f'cmd > {proc.args}') + cmdlog.error(f'expected return "{expects}" got "{proc.returncode}"') cmdlog.error('stdout') - for i in stdout: - cmdlog.error('{x}'.format(x=i)) + cmdlog.error(proc.stdout) cmdlog.error('stderr') - for i in stderr: - cmdlog.error('{x}'.format(x=i)) + cmdlog.error(proc.stderr) -# end run_cmd - def gen_data(size, rnd): chars = string.ascii_letters + string.digits return ''.join(rnd.choice(chars) for _ in range(size)) diff --git a/ceph/qa/workunits/rados/test_envlibrados_for_rocksdb.sh b/ceph/qa/workunits/rados/test_envlibrados_for_rocksdb.sh index ecbb4098a..ca9b8d2ea 100755 --- a/ceph/qa/workunits/rados/test_envlibrados_for_rocksdb.sh +++ b/ceph/qa/workunits/rados/test_envlibrados_for_rocksdb.sh @@ -33,7 +33,7 @@ case $(distro_id) in sudo subscription-manager repos --enable "codeready-builder-for-rhel-8-x86_64-rpms" ;; esac - install git gcc-c++.x86_64 snappy-devel zlib zlib-devel bzip2 bzip2-devel libradospp-devel.x86_64 cmake + install git gcc-c++.x86_64 snappy-devel zlib zlib-devel bzip2 bzip2-devel libradospp-devel.x86_64 cmake libarchive-3.3.3 ;; opensuse*|suse|sles) install git gcc-c++ snappy-devel zlib-devel libbz2-devel libradospp-devel diff --git a/ceph/qa/workunits/rbd/qemu-iotests.sh b/ceph/qa/workunits/rbd/qemu-iotests.sh index 1f13da9fc..267b69165 100755 --- a/ceph/qa/workunits/rbd/qemu-iotests.sh +++ b/ceph/qa/workunits/rbd/qemu-iotests.sh @@ -9,10 +9,12 @@ testlist='001 002 003 004 005 008 009 010 011 021 025 032 033' git clone https://github.com/qemu/qemu.git cd qemu -if lsb_release -da 2>&1 | grep -iqE '(bionic|focal)'; then + + +if grep -iqE '(bionic|focal)' /etc/os-release; then # Bionic requires a matching test harness git checkout v2.11.0 -elif lsb_release -da 2>&1 | grep -iqE '(xenial|linux release 8)'; then +elif grep -iqE '(xenial|platform:el8)' /etc/os-release; then # Xenial requires a recent test harness git checkout v2.3.0 else diff --git a/ceph/qa/workunits/rbd/rbd-nbd.sh b/ceph/qa/workunits/rbd/rbd-nbd.sh index 37718e3b5..d98a63ca6 100755 --- a/ceph/qa/workunits/rbd/rbd-nbd.sh +++ b/ceph/qa/workunits/rbd/rbd-nbd.sh @@ -4,6 +4,7 @@ set -ex . $(dirname $0)/../../standalone/ceph-helpers.sh POOL=rbd +ANOTHER_POOL=new_default_pool$$ NS=ns IMAGE=testrbdnbd$$ SIZE=64 @@ -56,6 +57,10 @@ setup() rbd --dest-pool ${POOL} --dest-namespace "${ns}" --no-progress import \ ${DATA} ${IMAGE} done + + # create another pool + ceph osd pool create ${ANOTHER_POOL} 8 + rbd pool init ${ANOTHER_POOL} } function cleanup() @@ -69,7 +74,7 @@ function cleanup() rm -Rf ${TEMPDIR} if [ -n "${DEV}" ] then - _sudo rbd-nbd unmap ${DEV} + _sudo rbd device --device-type nbd unmap ${DEV} fi for ns in '' ${NS}; do @@ -84,6 +89,10 @@ function cleanup() fi done rbd namespace remove ${POOL}/${NS} + + # cleanup/reset default pool + rbd config global rm global rbd_default_pool + ceph osd pool delete ${ANOTHER_POOL} ${ANOTHER_POOL} --yes-i-really-really-mean-it } function expect_false() @@ -93,10 +102,11 @@ function expect_false() function get_pid() { - local ns=$1 + local pool=$1 + local ns=$2 - PID=$(rbd-nbd --format xml list-mapped | $XMLSTARLET sel -t -v \ - "//devices/device[pool='${POOL}'][namespace='${ns}'][image='${IMAGE}'][device='${DEV}']/id") + PID=$(rbd device --device-type nbd --format xml list | $XMLSTARLET sel -t -v \ + "//devices/device[pool='${pool}'][namespace='${ns}'][image='${IMAGE}'][device='${DEV}']/id") test -n "${PID}" || return 1 ps -p ${PID} -C rbd-nbd } @@ -106,8 +116,8 @@ unmap_device() local dev=$1 local pid=$2 - _sudo rbd-nbd unmap ${dev} - rbd-nbd list-mapped | expect_false grep "^${pid}\\b" || return 1 + _sudo rbd device --device-type nbd unmap ${dev} + rbd device --device-type nbd list | expect_false grep "^${pid}\\b" || return 1 ps -C rbd-nbd | expect_false grep "^ *${pid}\\b" || return 1 # workaround possible race between unmap and following map @@ -125,19 +135,19 @@ expect_false rbd-nbd expect_false rbd-nbd INVALIDCMD if [ `id -u` -ne 0 ] then - expect_false rbd-nbd map ${IMAGE} + expect_false rbd device --device-type nbd map ${IMAGE} fi -expect_false _sudo rbd-nbd map INVALIDIMAGE +expect_false _sudo rbd device --device-type nbd map INVALIDIMAGE expect_false _sudo rbd-nbd --device INVALIDDEV map ${IMAGE} # list format test -expect_false rbd-nbd --format INVALID list-mapped -rbd-nbd --format json --pretty-format list-mapped -rbd-nbd --format xml list-mapped +expect_false rbd device --device-type nbd --format INVALID list +rbd device --device-type nbd --format json --pretty-format list +rbd device --device-type nbd --format xml list # map test using the first unused device -DEV=`_sudo rbd-nbd map ${POOL}/${IMAGE}` -get_pid +DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}` +get_pid ${POOL} # map test specifying the device expect_false _sudo rbd-nbd --device ${DEV} map ${POOL}/${IMAGE} dev1=${DEV} @@ -146,8 +156,8 @@ DEV= # XXX: race possible when the device is reused by other process DEV=`_sudo rbd-nbd --device ${dev1} map ${POOL}/${IMAGE}` [ "${DEV}" = "${dev1}" ] -rbd-nbd list-mapped | grep "${IMAGE}" -get_pid +rbd device --device-type nbd list | grep "${IMAGE}" +get_pid ${POOL} # read test [ "`dd if=${DATA} bs=1M | md5sum`" = "`_sudo dd if=${DEV} bs=1M | md5sum`" ] @@ -187,8 +197,8 @@ test ${blocks2} -eq ${blocks} # read-only option test unmap_device ${DEV} ${PID} -DEV=`_sudo rbd-nbd map --read-only ${POOL}/${IMAGE}` -PID=$(rbd-nbd list-mapped | awk -v pool=${POOL} -v img=${IMAGE} -v dev=${DEV} \ +DEV=`_sudo rbd --device-type nbd map --read-only ${POOL}/${IMAGE}` +PID=$(rbd device --device-type nbd list | awk -v pool=${POOL} -v img=${IMAGE} -v dev=${DEV} \ '$2 == pool && $3 == img && $5 == dev {print $1}') test -n "${PID}" ps -p ${PID} -C rbd-nbd @@ -198,8 +208,8 @@ expect_false _sudo dd if=${DATA} of=${DEV} bs=1M oflag=direct unmap_device ${DEV} ${PID} # exclusive option test -DEV=`_sudo rbd-nbd map --exclusive ${POOL}/${IMAGE}` -get_pid +DEV=`_sudo rbd --device-type nbd map --exclusive ${POOL}/${IMAGE}` +get_pid ${POOL} _sudo dd if=${DATA} of=${DEV} bs=1M oflag=direct expect_false timeout 10 \ @@ -209,49 +219,68 @@ DEV= rbd bench ${IMAGE} --io-type write --io-size=1024 --io-total=1024 # unmap by image name test -DEV=`_sudo rbd-nbd map ${POOL}/${IMAGE}` -get_pid +DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}` +get_pid ${POOL} unmap_device ${IMAGE} ${PID} DEV= # map/unmap snap test rbd snap create ${POOL}/${IMAGE}@snap -DEV=`_sudo rbd-nbd map ${POOL}/${IMAGE}@snap` -get_pid +DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}@snap` +get_pid ${POOL} unmap_device "${IMAGE}@snap" ${PID} DEV= # map/unmap namespace test rbd snap create ${POOL}/${NS}/${IMAGE}@snap -DEV=`_sudo rbd-nbd map ${POOL}/${NS}/${IMAGE}@snap` -get_pid ${NS} +DEV=`_sudo rbd device --device-type nbd map ${POOL}/${NS}/${IMAGE}@snap` +get_pid ${POOL} ${NS} unmap_device "${POOL}/${NS}/${IMAGE}@snap" ${PID} DEV= # unmap by image name test 2 -DEV=`_sudo rbd-nbd map ${POOL}/${IMAGE}` -get_pid +DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}` +get_pid ${POOL} pid=$PID -DEV=`_sudo rbd-nbd map ${POOL}/${NS}/${IMAGE}` -get_pid ${NS} +DEV=`_sudo rbd device --device-type nbd map ${POOL}/${NS}/${IMAGE}` +get_pid ${POOL} ${NS} unmap_device ${POOL}/${NS}/${IMAGE} ${PID} DEV= unmap_device ${POOL}/${IMAGE} ${pid} +# map/unmap test with just image name and expect image to come from default pool +if [ "${POOL}" = "rbd" ];then + DEV=`_sudo rbd device --device-type nbd map ${IMAGE}` + get_pid ${POOL} + unmap_device ${IMAGE} ${PID} + DEV= +fi + +# map/unmap test with just image name after changing default pool +rbd config global set global rbd_default_pool ${ANOTHER_POOL} +rbd create --size 10M ${IMAGE} +DEV=`_sudo rbd device --device-type nbd map ${IMAGE}` +get_pid ${ANOTHER_POOL} +unmap_device ${IMAGE} ${PID} +DEV= + +# reset +rbd config global rm global rbd_default_pool + # auto unmap test -DEV=`_sudo rbd-nbd map ${POOL}/${IMAGE}` -get_pid +DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}` +get_pid ${POOL} _sudo kill ${PID} for i in `seq 10`; do - rbd-nbd list-mapped | expect_false grep "^${PID} *${POOL} *${IMAGE}" && break + rbd device --device-type nbd list | expect_false grep "^${PID} *${POOL} *${IMAGE}" && break sleep 1 done -rbd-nbd list-mapped | expect_false grep "^${PID} *${POOL} *${IMAGE}" +rbd device --device-type nbd list | expect_false grep "^${PID} *${POOL} *${IMAGE}" # quiesce test QUIESCE_HOOK=${TEMPDIR}/quiesce.sh -DEV=`_sudo rbd-nbd map --quiesce --quiesce-hook ${QUIESCE_HOOK} ${POOL}/${IMAGE}` -get_pid +DEV=`_sudo rbd device --device-type nbd map --quiesce --quiesce-hook ${QUIESCE_HOOK} ${POOL}/${IMAGE}` +get_pid ${POOL} # test it fails if the hook does not exists test ! -e ${QUIESCE_HOOK} @@ -296,12 +325,12 @@ unmap_device ${DEV} ${PID} LOG_FILE=${TEMPDIR}/rbd-nbd.log if [ -n "${CEPH_SRC}" ]; then QUIESCE_HOOK=${CEPH_SRC}/tools/rbd_nbd/rbd-nbd_quiesce - DEV=`_sudo rbd-nbd map --quiesce --quiesce-hook ${QUIESCE_HOOK} \ + DEV=`_sudo rbd device --device-type nbd map --quiesce --quiesce-hook ${QUIESCE_HOOK} \ ${POOL}/${IMAGE} --log-file=${LOG_FILE}` else - DEV=`_sudo rbd-nbd map --quiesce ${POOL}/${IMAGE} --log-file=${LOG_FILE}` + DEV=`_sudo rbd device --device-type nbd map --quiesce ${POOL}/${IMAGE} --log-file=${LOG_FILE}` fi -get_pid +get_pid ${POOL} _sudo mkfs ${DEV} mkdir ${TEMPDIR}/mnt _sudo mount ${DEV} ${TEMPDIR}/mnt @@ -314,17 +343,17 @@ cat ${LOG_FILE} expect_false grep 'quiesce failed' ${LOG_FILE} # test detach/attach -DEV=`_sudo rbd-nbd map --try-netlink ${POOL}/${IMAGE}` -get_pid +DEV=`_sudo rbd device --device-type nbd --options try-netlink map ${POOL}/${IMAGE}` +get_pid ${POOL} _sudo mount ${DEV} ${TEMPDIR}/mnt _sudo rbd-nbd detach ${POOL}/${IMAGE} -expect_false get_pid +expect_false get_pid ${POOL} _sudo rbd-nbd attach --device ${DEV} ${POOL}/${IMAGE} -get_pid +get_pid ${POOL} _sudo rbd-nbd detach ${DEV} -expect_false get_pid +expect_false get_pid ${POOL} _sudo rbd-nbd attach --device ${DEV} ${POOL}/${IMAGE} -get_pid +get_pid ${POOL} ls ${TEMPDIR}/mnt/ dd if=${TEMPDIR}/mnt/test of=/dev/null bs=1M count=1 _sudo dd if=${DATA} of=${TEMPDIR}/mnt/test1 bs=1M count=1 oflag=direct diff --git a/ceph/qa/workunits/rgw/s3_utilities.pm b/ceph/qa/workunits/rgw/s3_utilities.pm index 12e6af0ad..3c3fae900 100644 --- a/ceph/qa/workunits/rgw/s3_utilities.pm +++ b/ceph/qa/workunits/rgw/s3_utilities.pm @@ -134,15 +134,28 @@ sub purge_data return 0; } +# Read PRETTY_NAME from /etc/os-release +sub os_pretty_name +{ + open(FH, '<', '/etc/os-release') or die $!; + while (my $line = ) { + chomp $line; + if ($line =~ /^\s*PRETTY_NAME=\"?([^"]*)\"?/) { + return $1; + } + } + close(FH); +} + + # Function to get the Ceph and distro info sub ceph_os_info { my $ceph_v = get_command_output ( "ceph -v" ); my @ceph_arr = split(" ",$ceph_v); $ceph_v = "Ceph Version: $ceph_arr[2]"; - my $os_distro = get_command_output ( "lsb_release -d" ); - my @os_arr = split(":",$os_distro); - $os_distro = "Linux Flavor:$os_arr[1]"; + my $os_distro = os_pretty_name(); + $os_distro = "Linux Flavor:$os_distro"; return ($ceph_v, $os_distro); } diff --git a/ceph/run-make-check.sh b/ceph/run-make-check.sh index 896d2f634..f9fe99338 100755 --- a/ceph/run-make-check.sh +++ b/ceph/run-make-check.sh @@ -40,7 +40,7 @@ function run() { CHECK_MAKEOPTS=${CHECK_MAKEOPTS:-$DEFAULT_MAKEOPTS} if in_jenkins; then - if ! ctest $CHECK_MAKEOPTS --no-compress-output --output-on-failure -T Test; then + if ! ctest $CHECK_MAKEOPTS --no-compress-output --output-on-failure --test-output-size-failed 1024000 -T Test; then # do not return failure, as the jenkins publisher will take care of this rm -fr ${TMPDIR:-/tmp}/ceph-asok.* fi @@ -67,7 +67,7 @@ function main() { fi FOR_MAKE_CHECK=1 prepare # Init defaults after deps are installed. - local cmake_opts=" -DWITH_PYTHON3=3 -DWITH_GTEST_PARALLEL=ON -DWITH_FIO=ON -DWITH_CEPHFS_SHELL=ON -DWITH_SPDK=ON -DENABLE_GIT_VERSION=OFF" + local cmake_opts=" -DWITH_PYTHON3=3 -DWITH_GTEST_PARALLEL=ON -DWITH_FIO=ON -DWITH_CEPHFS_SHELL=ON -DWITH_GRAFANA=ON -DWITH_SPDK=ON -DENABLE_GIT_VERSION=OFF" if [ $WITH_SEASTAR ]; then cmake_opts+=" -DWITH_SEASTAR=ON" fi diff --git a/ceph/src/.git_version b/ceph/src/.git_version index 278f3aee5..43145fdd5 100644 --- a/ceph/src/.git_version +++ b/ceph/src/.git_version @@ -1,2 +1,2 @@ -0883bdea7337b95e4b611c768c0279868462204a -16.2.5 +ee28fb57e47e9f88813e24bbf4c14496ca299d31 +16.2.6 diff --git a/ceph/src/CMakeLists.txt b/ceph/src/CMakeLists.txt index 011408ceb..b6fa63366 100644 --- a/ceph/src/CMakeLists.txt +++ b/ceph/src/CMakeLists.txt @@ -304,12 +304,17 @@ add_subdirectory(json_spirit) include_directories(SYSTEM "${CMAKE_SOURCE_DIR}/src/xxHash") include_directories(SYSTEM "${CMAKE_SOURCE_DIR}/src/rapidjson/include") +option(WITH_FMT_HEADER_ONLY "use header-only version of fmt library" OFF) find_package(fmt 6.0.0 QUIET) if(fmt_FOUND) include_directories(SYSTEM "${fmt_INCLUDE_DIR}") else() message(STATUS "Could not find fmt, will build it") + set(old_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) + set(BUILD_SHARED_LIBS FALSE) add_subdirectory(fmt) + set(BUILD_SHARED_LIBS ${old_BUILD_SHARED_LIBS}) + unset(old_BUILD_SHARED_LIBS) include_directories(SYSTEM "${CMAKE_SOURCE_DIR}/src/fmt/include") endif() @@ -360,6 +365,15 @@ if(WITH_SEASTAR) add_subdirectory(crimson) endif() +function(compile_with_fmt target) + get_target_property(fmt_compile_definitions + fmt::fmt INTERFACE_COMPILE_DEFINITIONS) + if(fmt_compile_definitions) + target_compile_definitions(${target} PUBLIC + ${fmt_compile_definitions}) + endif() +endfunction() + set(libcommon_files ${CMAKE_BINARY_DIR}/src/include/ceph_ver.h ceph_ver.c @@ -396,6 +410,7 @@ endif() set_source_files_properties(ceph_ver.c APPEND PROPERTY OBJECT_DEPENDS ${CMAKE_BINARY_DIR}/src/include/ceph_ver.h) add_library(common-objs OBJECT ${libcommon_files}) +compile_with_fmt(common-objs) if(WITH_JAEGER) find_package(yaml-cpp 0.6.0) diff --git a/ceph/src/SimpleRADOSStriper.cc b/ceph/src/SimpleRADOSStriper.cc index 736e8769d..d6fe101f9 100644 --- a/ceph/src/SimpleRADOSStriper.cc +++ b/ceph/src/SimpleRADOSStriper.cc @@ -46,7 +46,7 @@ using ceph::bufferlist; -#define dout_subsys ceph_subsys_client +#define dout_subsys ceph_subsys_cephsqlite #undef dout_prefix #define dout_prefix *_dout << "client." << ioctx.get_instance_id() << ": SimpleRADOSStriper: " << __func__ << ": " << oid << ": " #define d(lvl) ldout((CephContext*)ioctx.cct(), (lvl)) diff --git a/ceph/src/blk/CMakeLists.txt b/ceph/src/blk/CMakeLists.txt index b424482de..849f3eef9 100644 --- a/ceph/src/blk/CMakeLists.txt +++ b/ceph/src/blk/CMakeLists.txt @@ -25,7 +25,7 @@ if(WITH_ZBD) zoned/HMSMRDevice.cc) endif() -add_library(blk ${libblk_srcs}) +add_library(blk STATIC ${libblk_srcs}) target_include_directories(blk PRIVATE "./") if(HAVE_LIBAIO) diff --git a/ceph/src/ceph-volume/ceph_volume/api/lvm.py b/ceph/src/ceph-volume/ceph_volume/api/lvm.py index 30362f1bd..0f38249e1 100644 --- a/ceph/src/ceph-volume/ceph_volume/api/lvm.py +++ b/ceph/src/ceph-volume/ceph_volume/api/lvm.py @@ -463,7 +463,7 @@ def get_pvs(fields=PV_FIELDS, filters='', tags=None): :returns: list of class PVolume object representing pvs on the system """ filters = make_filters_lvmcmd_ready(filters, tags) - args = ['pvs', '--no-heading', '--readonly', '--separator=";"', '-S', + args = ['pvs', '--noheadings', '--readonly', '--separator=";"', '-S', filters, '-o', fields] stdout, stderr, returncode = process.call(args, verbose_on_failure=False) @@ -1134,3 +1134,15 @@ def get_device_lvs(device, name_prefix=''): lvs = _output_parser(stdout, LV_FIELDS) return [Volume(**lv) for lv in lvs if lv['lv_name'] and lv['lv_name'].startswith(name_prefix)] + +def get_lv_by_fullname(full_name): + """ + returns LV by the specified LV's full name (formatted as vg_name/lv_name) + """ + try: + vg_name, lv_name = full_name.split('/') + res_lv = get_first_lv(filters={'lv_name': lv_name, + 'vg_name': vg_name}) + except ValueError: + res_lv = None + return res_lv diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py index e4ac074a4..c864b0e9f 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py @@ -269,6 +269,11 @@ class Activate(object): tags = {'ceph.osd_id': osd_id, 'ceph.osd_fsid': osd_fsid} elif not osd_id and osd_fsid: tags = {'ceph.osd_fsid': osd_fsid} + elif osd_id and not osd_fsid: + raise RuntimeError('could not activate osd.{}, please provide the ' + 'osd_fsid too'.format(osd_id)) + else: + raise RuntimeError('Please provide both osd_id and osd_fsid') lvs = api.get_lvs(tags=tags) if not lvs: raise RuntimeError('could not find osd.%s with osd_fsid %s' % diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/batch.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/batch.py index 114730ade..e64e4b64e 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/batch.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/batch.py @@ -512,10 +512,11 @@ class Batch(object): # fill up uneven distributions across fast devices: 5 osds and 2 fast # devices? create 3 slots on each device rather then deploying # heterogeneous osds - if (requested_osds - len(lvm_devs)) % len(phys_devs): - fast_slots_per_device = int((requested_osds - len(lvm_devs)) / len(phys_devs)) + 1 + slot_divider = max(1, len(phys_devs)) + if (requested_osds - len(lvm_devs)) % slot_divider: + fast_slots_per_device = int((requested_osds - len(lvm_devs)) / slot_divider) + 1 else: - fast_slots_per_device = int((requested_osds - len(lvm_devs)) / len(phys_devs)) + fast_slots_per_device = int((requested_osds - len(lvm_devs)) / slot_divider) ret.extend(get_physical_fast_allocs(phys_devs, diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/deactivate.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/deactivate.py index 5de6dbe36..46846a1dc 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/deactivate.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/deactivate.py @@ -54,8 +54,6 @@ class Deactivate(object): ceph-volume lvm deactivate {ID} {FSID} - To deactivate all volumes use the --all flag. - ceph-volume lvm deactivate --all """) parser = argparse.ArgumentParser( prog='ceph-volume lvm deactivate', diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/main.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/main.py index 3ef3c1117..39947454d 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/main.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/main.py @@ -9,6 +9,7 @@ from . import trigger from . import listing from . import zap from . import batch +from . import migrate class LVM(object): @@ -30,6 +31,9 @@ class LVM(object): 'trigger': trigger.Trigger, 'list': listing.List, 'zap': zap.Zap, + 'migrate': migrate.Migrate, + 'new-wal': migrate.NewWAL, + 'new-db': migrate.NewDB, } def __init__(self, argv): diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/migrate.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/migrate.py new file mode 100644 index 000000000..886b9f7b4 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/migrate.py @@ -0,0 +1,690 @@ +from __future__ import print_function +import argparse +import logging +import os +from textwrap import dedent +from ceph_volume.util import system, disk, merge_dict +from ceph_volume.util.device import Device +from ceph_volume import decorators, terminal, process +from ceph_volume.api import lvm as api +from ceph_volume.systemd import systemctl + + +logger = logging.getLogger(__name__) +mlogger = terminal.MultiLogger(__name__) + +def get_cluster_name(osd_id, osd_fsid): + """ + From an ``osd_id`` and/or an ``osd_fsid``, filter out all the LVs in the + system that match those tag values, then return cluster_name for the first + one. + """ + lv_tags = {} + lv_tags['ceph.osd_id'] = osd_id + lv_tags['ceph.osd_fsid'] = osd_fsid + + lvs = api.get_lvs(tags=lv_tags) + if not lvs: + mlogger.error( + 'Unable to find any LV for source OSD: id:{} fsid:{}'.format( + osd_id, osd_fsid) ) + raise SystemExit('Unexpected error, terminating') + return next(iter(lvs)).tags["ceph.cluster_name"] + +def get_osd_path(osd_id, osd_fsid): + return '/var/lib/ceph/osd/{}-{}'.format( + get_cluster_name(osd_id, osd_fsid), osd_id) + +def find_associated_devices(osd_id, osd_fsid): + """ + From an ``osd_id`` and/or an ``osd_fsid``, filter out all the LVs in the + system that match those tag values, further detect if any partitions are + part of the OSD, and then return the set of LVs and partitions (if any). + """ + lv_tags = {} + lv_tags['ceph.osd_id'] = osd_id + lv_tags['ceph.osd_fsid'] = osd_fsid + + lvs = api.get_lvs(tags=lv_tags) + if not lvs: + mlogger.error( + 'Unable to find any LV for source OSD: id:{} fsid:{}'.format( + osd_id, osd_fsid) ) + raise SystemExit('Unexpected error, terminating') + + devices = set(ensure_associated_lvs(lvs, lv_tags)) + return [(Device(path), type) for path, type in devices if path] + +def ensure_associated_lvs(lvs, lv_tags): + """ + Go through each LV and ensure if backing devices (journal, wal, block) + are LVs or partitions, so that they can be accurately reported. + """ + # look for many LVs for each backing type, because it is possible to + # receive a filtering for osd.1, and have multiple failed deployments + # leaving many journals with osd.1 - usually, only a single LV will be + # returned + + block_lvs = api.get_lvs(tags=merge_dict(lv_tags, {'ceph.type': 'block'})) + db_lvs = api.get_lvs(tags=merge_dict(lv_tags, {'ceph.type': 'db'})) + wal_lvs = api.get_lvs(tags=merge_dict(lv_tags, {'ceph.type': 'wal'})) + backing_devices = [(block_lvs, 'block'), (db_lvs, 'db'), + (wal_lvs, 'wal')] + + verified_devices = [] + + for lv in lvs: + # go through each lv and append it, otherwise query `blkid` to find + # a physical device. Do this for each type (journal,db,wal) regardless + # if they have been processed in the previous LV, so that bad devices + # with the same ID can be caught + for ceph_lvs, type in backing_devices: + + if ceph_lvs: + verified_devices.extend([(l.lv_path, type) for l in ceph_lvs]) + continue + + # must be a disk partition, by querying blkid by the uuid we are + # ensuring that the device path is always correct + try: + device_uuid = lv.tags['ceph.{}_uuid'.format(type)] + except KeyError: + # Bluestore will not have ceph.journal_uuid, and Filestore + # will not not have ceph.db_uuid + continue + + osd_device = disk.get_device_from_partuuid(device_uuid) + if not osd_device: + # if the osd_device is not found by the partuuid, then it is + # not possible to ensure this device exists anymore, so skip it + continue + verified_devices.append((osd_device, type)) + + return verified_devices + +class VolumeTagTracker(object): + def __init__(self, devices, target_lv): + self.target_lv = target_lv + self.data_device = self.db_device = self.wal_device = None + for device, type in devices: + if type == 'block': + self.data_device = device + elif type == 'db': + self.db_device = device + elif type == 'wal': + self.wal_device = device + if not self.data_device: + mlogger.error('Data device not found') + raise SystemExit( + "Unexpected error, terminating") + if not self.data_device.is_lv: + mlogger.error('Data device isn\'t LVM') + raise SystemExit( + "Unexpected error, terminating") + + self.old_target_tags = self.target_lv.tags.copy() + self.old_data_tags = ( + self.data_device.lv_api.tags.copy() + if self.data_device.is_lv else None) + self.old_db_tags = ( + self.db_device.lv_api.tags.copy() + if self.db_device and self.db_device.is_lv else None) + self.old_wal_tags = ( + self.wal_device.lv_api.tags.copy() + if self.wal_device and self.wal_device.is_lv else None) + + def update_tags_when_lv_create(self, create_type): + tags = {} + if not self.data_device.is_lv: + mlogger.warning( + 'Data device is not LVM, wouldn\'t update LVM tags') + else: + tags["ceph.{}_uuid".format(create_type)] = self.target_lv.lv_uuid + tags["ceph.{}_device".format(create_type)] = self.target_lv.lv_path + self.data_device.lv_api.set_tags(tags) + + tags = self.data_device.lv_api.tags.copy() + tags["ceph.type"] = create_type + self.target_lv.set_tags(tags) + + aux_dev = None + if create_type == "db" and self.wal_device: + aux_dev = self.wal_device + elif create_type == "wal" and self.db_device: + aux_dev = self.db_device + else: + return + if not aux_dev.is_lv: + mlogger.warning( + '{} device is not LVM, wouldn\'t update LVM tags'.format( + create_type.upper())) + else: + tags = {} + tags["ceph.{}_uuid".format(create_type)] = self.target_lv.lv_uuid + tags["ceph.{}_device".format(create_type)] = self.target_lv.lv_path + aux_dev.lv_api.set_tags(tags) + + def remove_lvs(self, source_devices, target_type): + remaining_devices = [self.data_device, self.db_device, self.wal_device] + + outdated_tags = [] + for device, type in source_devices: + if type == "block" or type == target_type: + continue + remaining_devices.remove(device) + if device.is_lv: + outdated_tags.append("ceph.{}_uuid".format(type)) + outdated_tags.append("ceph.{}_device".format(type)) + device.lv_api.clear_tags() + if len(outdated_tags) > 0: + for d in remaining_devices: + if d and d.is_lv: + d.lv_api.clear_tags(outdated_tags) + + def replace_lvs(self, source_devices, target_type): + remaining_devices = [self.data_device] + if self.db_device: + remaining_devices.append(self.db_device) + if self.wal_device: + remaining_devices.append(self.wal_device) + + outdated_tags = [] + for device, type in source_devices: + if type == "block": + continue + remaining_devices.remove(device) + if device.is_lv: + outdated_tags.append("ceph.{}_uuid".format(type)) + outdated_tags.append("ceph.{}_device".format(type)) + device.lv_api.clear_tags() + + new_tags = {} + new_tags["ceph.{}_uuid".format(target_type)] = self.target_lv.lv_uuid + new_tags["ceph.{}_device".format(target_type)] = self.target_lv.lv_path + + for d in remaining_devices: + if d and d.is_lv: + if len(outdated_tags) > 0: + d.lv_api.clear_tags(outdated_tags) + d.lv_api.set_tags(new_tags) + + if not self.data_device.is_lv: + mlogger.warning( + 'Data device is not LVM, wouldn\'t properly update target LVM tags') + else: + tags = self.data_device.lv_api.tags.copy() + + tags["ceph.type"] = target_type + tags["ceph.{}_uuid".format(target_type)] = self.target_lv.lv_uuid + tags["ceph.{}_device".format(target_type)] = self.target_lv.lv_path + self.target_lv.set_tags(tags) + + def undo(self): + mlogger.info( + 'Undoing lv tag set') + if self.data_device: + if self.old_data_tags: + self.data_device.lv_api.set_tags(self.old_data_tags) + else: + self.data_device.lv_api.clear_tags() + if self.db_device: + if self.old_db_tags: + self.db_device.lv_api.set_tags(self.old_db_tags) + else: + self.db_device.lv_api.clear_tags() + if self.wal_device: + if self.old_wal_tags: + self.wal_device.lv_api.set_tags(self.old_wal_tags) + else: + self.wal_device.lv_api.clear_tags() + if self.old_target_tags: + self.target_lv.set_tags(self.old_target_tags) + else: + self.target_lv.clear_tags() + +class Migrate(object): + + help = 'Migrate BlueFS data from to another LVM device' + + def __init__(self, argv): + self.argv = argv + self.osd_id = None + + def get_source_devices(self, devices, target_type=""): + ret = [] + for device, type in devices: + if type == target_type: + continue + if type == 'block': + if 'data' not in self.args.from_: + continue; + elif type == 'db': + if 'db' not in self.args.from_: + continue; + elif type == 'wal': + if 'wal' not in self.args.from_: + continue; + ret.append([device, type]) + if ret == []: + mlogger.error('Source device list is empty') + raise SystemExit( + 'Unable to migrate to : {}'.format(self.args.target)) + return ret + + # ceph-bluestore-tool uses the following replacement rules + # (in the order of precedence, stop on the first match) + # if source list has DB volume - target device replaces it. + # if source list has WAL volume - target device replace it. + # if source list has slow volume only - operation isn’t permitted, + # requires explicit allocation via new-db/new-wal command.detects which + def get_target_type_by_source(self, devices): + ret = None + for device, type in devices: + if type == 'db': + return 'db' + elif type == 'wal': + ret = 'wal' + return ret + + def get_filename_by_type(self, type): + filename = 'block' + if type == 'db' or type == 'wal': + filename += '.' + type + return filename + + def get_source_args(self, osd_path, devices): + ret = [] + for device, type in devices: + ret = ret + ["--devs-source", os.path.join( + osd_path, self.get_filename_by_type(type))] + return ret + + @decorators.needs_root + def migrate_to_new(self, osd_id, osd_fsid, devices, target_lv): + source_devices = self.get_source_devices(devices) + target_type = self.get_target_type_by_source(source_devices) + if not target_type: + mlogger.error( + "Unable to determine new volume type," + " please use new-db or new-wal command before.") + raise SystemExit( + "Unable to migrate to : {}".format(self.args.target)) + + target_path = target_lv.lv_path + + try: + tag_tracker = VolumeTagTracker(devices, target_lv) + # we need to update lvm tags for all the remaining volumes + # and clear for ones which to be removed + + # ceph-bluestore-tool removes source volume(s) other than block one + # and attaches target one after successful migration + tag_tracker.replace_lvs(source_devices, target_type) + + osd_path = get_osd_path(osd_id, osd_fsid) + source_args = self.get_source_args(osd_path, source_devices) + mlogger.info("Migrate to new, Source: {} Target: {}".format( + source_args, target_path)) + stdout, stderr, exit_code = process.call([ + 'ceph-bluestore-tool', + '--path', + osd_path, + '--dev-target', + target_path, + '--command', + 'bluefs-bdev-migrate'] + + source_args) + if exit_code != 0: + mlogger.error( + 'Failed to migrate device, error code:{}'.format(exit_code)) + raise SystemExit( + 'Failed to migrate to : {}'.format(self.args.target)) + else: + system.chown(os.path.join(osd_path, "block.{}".format( + target_type))) + terminal.success('Migration successful.') + except: + tag_tracker.undo() + raise + + return + + @decorators.needs_root + def migrate_to_existing(self, osd_id, osd_fsid, devices, target_lv): + target_type = target_lv.tags["ceph.type"] + if target_type == "wal": + mlogger.error("Migrate to WAL is not supported") + raise SystemExit( + "Unable to migrate to : {}".format(self.args.target)) + target_filename = self.get_filename_by_type(target_type) + if (target_filename == ""): + mlogger.error( + "Target Logical Volume doesn't have proper volume type " + "(ceph.type LVM tag): {}".format(target_type)) + raise SystemExit( + "Unable to migrate to : {}".format(self.args.target)) + + osd_path = get_osd_path(osd_id, osd_fsid) + source_devices = self.get_source_devices(devices, target_type) + target_path = os.path.join(osd_path, target_filename) + tag_tracker = VolumeTagTracker(devices, target_lv) + + try: + # ceph-bluestore-tool removes source volume(s) other than + # block and target ones after successful migration + tag_tracker.remove_lvs(source_devices, target_type) + source_args = self.get_source_args(osd_path, source_devices) + mlogger.info("Migrate to existing, Source: {} Target: {}".format( + source_args, target_path)) + stdout, stderr, exit_code = process.call([ + 'ceph-bluestore-tool', + '--path', + osd_path, + '--dev-target', + target_path, + '--command', + 'bluefs-bdev-migrate'] + + source_args) + if exit_code != 0: + mlogger.error( + 'Failed to migrate device, error code:{}'.format(exit_code)) + raise SystemExit( + 'Failed to migrate to : {}'.format(self.args.target)) + else: + terminal.success('Migration successful.') + except: + tag_tracker.undo() + raise + + return + + @decorators.needs_root + def migrate_osd(self): + if self.args.osd_id and not self.args.no_systemd: + osd_is_running = systemctl.osd_is_active(self.args.osd_id) + if osd_is_running: + mlogger.error('OSD is running, stop it with: ' + 'systemctl stop ceph-osd@{}'.format( + self.args.osd_id)) + raise SystemExit( + 'Unable to migrate devices associated with OSD ID: {}' + .format(self.args.osd_id)) + + target_lv = api.get_lv_by_fullname(self.args.target) + if not target_lv: + mlogger.error( + 'Target path "{}" is not a Logical Volume'.formaat( + self.args.target)) + raise SystemExit( + 'Unable to migrate to : {}'.format(self.args.target)) + devices = find_associated_devices(self.args.osd_id, self.args.osd_fsid) + if (not target_lv.used_by_ceph): + self.migrate_to_new(self.args.osd_id, self.args.osd_fsid, + devices, + target_lv) + else: + if (target_lv.tags['ceph.osd_id'] != self.args.osd_id or + target_lv.tags['ceph.osd_fsid'] != self.args.osd_fsid): + mlogger.error( + 'Target Logical Volume isn\'t used by the specified OSD: ' + '{} FSID: {}'.format(self.args.osd_id, + self.args.osd_fsid)) + raise SystemExit( + 'Unable to migrate to : {}'.format(self.args.target)) + + self.migrate_to_existing(self.args.osd_id, self.args.osd_fsid, + devices, + target_lv) + + def make_parser(self, prog, sub_command_help): + parser = argparse.ArgumentParser( + prog=prog, + formatter_class=argparse.RawDescriptionHelpFormatter, + description=sub_command_help, + ) + + parser.add_argument( + '--osd-id', + required=True, + help='Specify an OSD ID to detect associated devices for zapping', + ) + + parser.add_argument( + '--osd-fsid', + required=True, + help='Specify an OSD FSID to detect associated devices for zapping', + ) + parser.add_argument( + '--target', + required=True, + help='Specify target Logical Volume (LV) to migrate data to', + ) + parser.add_argument( + '--from', + nargs='*', + dest='from_', + required=True, + choices=['data', 'db', 'wal'], + help='Copy BlueFS data from DB device', + ) + parser.add_argument( + '--no-systemd', + dest='no_systemd', + action='store_true', + help='Skip checking OSD systemd unit', + ) + return parser + + def main(self): + sub_command_help = dedent(""" + Moves BlueFS data from source volume(s) to the target one, source + volumes (except the main (i.e. data or block) one) are removed on + success. LVM volumes are permitted for Target only, both already + attached or new logical one. In the latter case it is attached to OSD + replacing one of the source devices. Following replacement rules apply + (in the order of precedence, stop on the first match): + * if source list has DB volume - target device replaces it. + * if source list has WAL volume - target device replace it. + * if source list has slow volume only - operation is not permitted, + requires explicit allocation via new-db/new-wal command. + + Example calls for supported scenarios: + + Moves BlueFS data from main device to LV already attached as DB: + + ceph-volume lvm migrate --osd-id 1 --osd-fsid --from data --target vgname/db + + Moves BlueFS data from shared main device to LV which will be attached + as a new DB: + + ceph-volume lvm migrate --osd-id 1 --osd-fsid --from data --target vgname/new_db + + Moves BlueFS data from DB device to new LV, DB is replaced: + + ceph-volume lvm migrate --osd-id 1 --osd-fsid --from db --target vgname/new_db + + Moves BlueFS data from main and DB devices to new LV, DB is replaced: + + ceph-volume lvm migrate --osd-id 1 --osd-fsid --from data db --target vgname/new_db + + Moves BlueFS data from main, DB and WAL devices to new LV, WAL is + removed and DB is replaced: + + ceph-volume lvm migrate --osd-id 1 --osd-fsid --from data db wal --target vgname/new_db + + Moves BlueFS data from main, DB and WAL devices to main device, WAL + and DB are removed: + + ceph-volume lvm migrate --osd-id 1 --osd-fsid --from db wal --target vgname/data + + """) + + parser = self.make_parser('ceph-volume lvm migrate', sub_command_help) + + if len(self.argv) == 0: + print(sub_command_help) + return + + self.args = parser.parse_args(self.argv) + + self.migrate_osd() + +class NewVolume(object): + def __init__(self, create_type, argv): + self.create_type = create_type + self.argv = argv + + def make_parser(self, prog, sub_command_help): + parser = argparse.ArgumentParser( + prog=prog, + formatter_class=argparse.RawDescriptionHelpFormatter, + description=sub_command_help, + ) + + parser.add_argument( + '--osd-id', + required=True, + help='Specify an OSD ID to attach new volume to', + ) + + parser.add_argument( + '--osd-fsid', + required=True, + help='Specify an OSD FSIDto attach new volume to', + ) + parser.add_argument( + '--target', + required=True, + help='Specify target Logical Volume (LV) to attach', + ) + parser.add_argument( + '--no-systemd', + dest='no_systemd', + action='store_true', + help='Skip checking OSD systemd unit', + ) + return parser + + @decorators.needs_root + def make_new_volume(self, osd_id, osd_fsid, devices, target_lv): + osd_path = get_osd_path(osd_id, osd_fsid) + mlogger.info( + 'Making new volume at {} for OSD: {} ({})'.format( + target_lv.lv_path, osd_id, osd_path)) + tag_tracker = VolumeTagTracker(devices, target_lv) + + try: + tag_tracker.update_tags_when_lv_create(self.create_type) + + stdout, stderr, exit_code = process.call([ + 'ceph-bluestore-tool', + '--path', + osd_path, + '--dev-target', + target_lv.lv_path, + '--command', + 'bluefs-bdev-new-{}'.format(self.create_type) + ]) + if exit_code != 0: + mlogger.error( + 'failed to attach new volume, error code:{}'.format( + exit_code)) + raise SystemExit( + "Failed to attach new volume: {}".format( + self.args.target)) + else: + system.chown(os.path.join(osd_path, "block.{}".format( + self.create_type))) + terminal.success('New volume attached.') + except: + tag_tracker.undo() + raise + return + + @decorators.needs_root + def new_volume(self): + if self.args.osd_id and not self.args.no_systemd: + osd_is_running = systemctl.osd_is_active(self.args.osd_id) + if osd_is_running: + mlogger.error('OSD ID is running, stop it with:' + ' systemctl stop ceph-osd@{}'.format(self.args.osd_id)) + raise SystemExit( + 'Unable to attach new volume for OSD: {}'.format( + self.args.osd_id)) + + target_lv = api.get_lv_by_fullname(self.args.target) + if not target_lv: + mlogger.error( + 'Target path {} is not a Logical Volume'.format( + self.args.target)) + raise SystemExit( + 'Unable to attach new volume : {}'.format(self.args.target)) + if target_lv.used_by_ceph: + mlogger.error( + 'Target Logical Volume is already used by ceph: {}'.format( + self.args.target)) + raise SystemExit( + 'Unable to attach new volume : {}'.format(self.args.target)) + else: + devices = find_associated_devices(self.args.osd_id, + self.args.osd_fsid) + self.make_new_volume( + self.args.osd_id, + self.args.osd_fsid, + devices, + target_lv) + +class NewWAL(NewVolume): + + help = 'Allocate new WAL volume for OSD at specified Logical Volume' + + def __init__(self, argv): + super(NewWAL, self).__init__("wal", argv) + + def main(self): + sub_command_help = dedent(""" + Attaches the given logical volume to the given OSD as a WAL volume. + Logical volume format is vg/lv. Fails if OSD has already got attached DB. + + Example: + + Attach vgname/lvname as a WAL volume to OSD 1 + + ceph-volume lvm new-wal --osd-id 1 --osd-fsid 55BD4219-16A7-4037-BC20-0F158EFCC83D --target vgname/new_wal + """) + parser = self.make_parser('ceph-volume lvm new-wal', sub_command_help) + + if len(self.argv) == 0: + print(sub_command_help) + return + + self.args = parser.parse_args(self.argv) + + self.new_volume() + +class NewDB(NewVolume): + + help = 'Allocate new DB volume for OSD at specified Logical Volume' + + def __init__(self, argv): + super(NewDB, self).__init__("db", argv) + + def main(self): + sub_command_help = dedent(""" + Attaches the given logical volume to the given OSD as a DB volume. + Logical volume format is vg/lv. Fails if OSD has already got attached DB. + + Example: + + Attach vgname/lvname as a DB volume to OSD 1 + + ceph-volume lvm new-db --osd-id 1 --osd-fsid 55BD4219-16A7-4037-BC20-0F158EFCC83D --target vgname/new_db + """) + + parser = self.make_parser('ceph-volume lvm new-db', sub_command_help) + if len(self.argv) == 0: + print(sub_command_help) + return + self.args = parser.parse_args(self.argv) + + self.new_volume() diff --git a/ceph/src/ceph-volume/ceph_volume/devices/raw/list.py b/ceph/src/ceph-volume/ceph_volume/devices/raw/list.py index 777930e98..c86d4996f 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/raw/list.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/raw/list.py @@ -4,10 +4,12 @@ import json import logging from textwrap import dedent from ceph_volume import decorators, process +from ceph_volume.util import disk logger = logging.getLogger(__name__) + def direct_report(devices): """ Other non-cli consumers of listing information will want to consume the @@ -18,6 +20,40 @@ def direct_report(devices): _list = List([]) return _list.generate(devices) +def _get_bluestore_info(dev): + out, err, rc = process.call([ + 'ceph-bluestore-tool', 'show-label', + '--dev', dev], verbose_on_failure=False) + if rc: + # ceph-bluestore-tool returns an error (below) if device is not bluestore OSD + # > unable to read label for : (2) No such file or directory + # but it's possible the error could be for a different reason (like if the disk fails) + logger.debug('assuming device {} is not BlueStore; ceph-bluestore-tool failed to get info from device: {}\n{}'.format(dev, out, err)) + return None + oj = json.loads(''.join(out)) + if dev not in oj: + # should be impossible, so warn + logger.warning('skipping device {} because it is not reported in ceph-bluestore-tool output: {}'.format(dev, out)) + return None + try: + if oj[dev]['description'] != 'main': + # ignore non-main devices, for now + logger.info('ignoring non-main device {}'.format(dev)) + return None + whoami = oj[dev]['whoami'] + return { + 'type': 'bluestore', + 'osd_id': int(whoami), + 'osd_uuid': oj[dev]['osd_uuid'], + 'ceph_fsid': oj[dev]['ceph_fsid'], + 'device': dev + } + except KeyError as e: + # this will appear for devices that have a bluestore header but aren't valid OSDs + # for example, due to incomplete rollback of OSDs: https://tracker.ceph.com/issues/51869 + logger.error('device {} does not have all BlueStore data needed to be a valid OSD: {}\n{}'.format(dev, out, e)) + return None + class List(object): @@ -27,64 +63,53 @@ class List(object): self.argv = argv def generate(self, devs=None): - if not devs: - logger.debug('Listing block devices via lsblk...') + logger.debug('Listing block devices via lsblk...') + + if devs is None or devs == []: devs = [] - # adding '--inverse' allows us to get the mapper devices list in that command output. - # not listing root devices containing partitions shouldn't have side effect since we are - # in `ceph-volume raw` context. - # - # example: - # running `lsblk --paths --nodeps --output=NAME --noheadings` doesn't allow to get the mapper list - # because the output is like following : - # - # $ lsblk --paths --nodeps --output=NAME --noheadings - # /dev/sda - # /dev/sdb - # /dev/sdc - # /dev/sdd - # - # the dmcrypt mappers are hidden because of the `--nodeps` given they are displayed as a dependency. - # - # $ lsblk --paths --output=NAME --noheadings - # /dev/sda - # |-/dev/mapper/ceph-3b52c90d-6548-407d-bde1-efd31809702f-sda-block-dmcrypt - # `-/dev/mapper/ceph-3b52c90d-6548-407d-bde1-efd31809702f-sda-db-dmcrypt - # /dev/sdb - # /dev/sdc - # /dev/sdd - # - # adding `--inverse` is a trick to get around this issue, the counterpart is that we can't list root devices if they contain - # at least one partition but this shouldn't be an issue in `ceph-volume raw` context given we only deal with raw devices. + # If no devs are given initially, we want to list ALL devices including children and + # parents. Parent disks with child partitions may be the appropriate device to return if + # the parent disk has a bluestore header, but children may be the most appropriate + # devices to return if the parent disk does not have a bluestore header. out, err, ret = process.call([ - 'lsblk', '--paths', '--nodeps', '--output=NAME', '--noheadings', '--inverse' + 'lsblk', '--paths', '--output=NAME', '--noheadings', '--list' ]) assert not ret devs = out + result = {} + logger.debug('inspecting devices: {}'.format(devs)) for dev in devs: - logger.debug('Examining %s' % dev) - # bluestore? - out, err, ret = process.call([ - 'ceph-bluestore-tool', 'show-label', - '--dev', dev], verbose_on_failure=False) - if ret: - logger.debug('No label on %s' % dev) - continue - oj = json.loads(''.join(out)) - if dev not in oj: + info = disk.lsblk(dev, abspath=True) + # Linux kernels built with CONFIG_ATARI_PARTITION enabled can falsely interpret + # bluestore's on-disk format as an Atari partition table. These false Atari partitions + # can be interpreted as real OSDs if a bluestore OSD was previously created on the false + # partition. See https://tracker.ceph.com/issues/52060 for more info. If a device has a + # parent, it is a child. If the parent is a valid bluestore OSD, the child will only + # exist if it is a phantom Atari partition, and the child should be ignored. If the + # parent isn't bluestore, then the child could be a valid bluestore OSD. If we fail to + # determine whether a parent is bluestore, we should err on the side of not reporting + # the child so as not to give a false negative. + if 'PKNAME' in info and info['PKNAME'] != "": + parent = info['PKNAME'] + try: + if disk.has_bluestore_label(parent): + logger.warning(('ignoring child device {} whose parent {} is a BlueStore OSD.'.format(dev, parent), + 'device is likely a phantom Atari partition. device info: {}'.format(info))) + continue + except OSError as e: + logger.error(('ignoring child device {} to avoid reporting invalid BlueStore data from phantom Atari partitions.'.format(dev), + 'failed to determine if parent device {} is BlueStore. err: {}'.format(parent, e))) + continue + + bs_info = _get_bluestore_info(dev) + if bs_info is None: + # None is also returned in the rare event that there is an issue reading info from + # a BlueStore disk, so be sure to log our assumption that it isn't bluestore + logger.info('device {} does not have BlueStore information'.format(dev)) continue - if oj[dev]['description'] != 'main': - # ignore non-main devices, for now - continue - whoami = oj[dev]['whoami'] - result[oj[dev]['osd_uuid']] = { - 'type': 'bluestore', - 'osd_id': int(whoami), - 'osd_uuid': oj[dev]['osd_uuid'], - 'ceph_fsid': oj[dev]['ceph_fsid'], - 'device': dev - } + result[bs_info['osd_uuid']] = bs_info + return result @decorators.needs_root diff --git a/ceph/src/ceph-volume/ceph_volume/tests/conftest.py b/ceph/src/ceph-volume/ceph_volume/tests/conftest.py index 2abedac32..149afbbc6 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/conftest.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/conftest.py @@ -52,6 +52,8 @@ def mock_lv_device_generator(): dev.used_by_ceph = False dev.vg_size = [size] dev.vg_free = dev.vg_size + dev.available_lvm = True + dev.is_device = False dev.lvs = [lvm.Volume(vg_name=dev.vg_name, lv_name=dev.lv_name, lv_size=size, lv_tags='')] return dev return mock_lv diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py index 33e0ed32b..46d7c3c83 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py @@ -58,6 +58,18 @@ class TestActivate(object): with pytest.raises(RuntimeError): activate.Activate([]).activate(args) + def test_osd_id_no_osd_fsid(self, is_root): + args = Args(osd_id=42, osd_fsid=None) + with pytest.raises(RuntimeError) as result: + activate.Activate([]).activate(args) + assert result.value.args[0] == 'could not activate osd.42, please provide the osd_fsid too' + + def test_no_osd_id_no_osd_fsid(self, is_root): + args = Args(osd_id=None, osd_fsid=None) + with pytest.raises(RuntimeError) as result: + activate.Activate([]).activate(args) + assert result.value.args[0] == 'Please provide both osd_id and osd_fsid' + def test_filestore_no_systemd(self, is_root, monkeypatch, capture): monkeypatch.setattr('ceph_volume.configuration.load', lambda: None) fake_enable = Capture() diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_batch.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_batch.py index 7c968ae81..4bf026ae1 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_batch.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_batch.py @@ -209,6 +209,15 @@ class TestBatch(object): 'block_db', 2, 2, args) assert len(fast) == 2 + def test_batch_fast_allocations_one_block_db_length(self, factory, conf_ceph_stub, + mock_lv_device_generator): + conf_ceph_stub('[global]\nfsid=asdf-lkjh') + + b = batch.Batch([]) + db_lv_devices = [mock_lv_device_generator()] + fast = b.fast_allocations(db_lv_devices, 1, 0, 'block_db') + assert len(fast) == 1 + @pytest.mark.parametrize('occupied_prior', range(7)) @pytest.mark.parametrize('slots,num_devs', [l for sub in [list(zip([x]*x, range(1, x + 1))) for x in range(1,7)] for l in sub]) diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_migrate.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_migrate.py new file mode 100644 index 000000000..6c2027350 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_migrate.py @@ -0,0 +1,2295 @@ +import pytest +from mock.mock import patch +from ceph_volume import process +from ceph_volume.api import lvm as api +from ceph_volume.devices.lvm import migrate +from ceph_volume.util.device import Device +from ceph_volume.util import system + +class TestGetClusterName(object): + + mock_volumes = [] + def mock_get_lvs(self, *args, **kwargs): + return self.mock_volumes.pop(0) + + def test_cluster_found(self, monkeypatch): + tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data,ceph.osd_fsid=1234,ceph.cluster_name=name_of_the_cluster' + vol = api.Volume(lv_name='volume1', lv_uuid='y', vg_name='', + lv_path='/dev/VolGroup/lv1', lv_tags=tags) + self.mock_volumes = [] + self.mock_volumes.append([vol]) + + monkeypatch.setattr(migrate.api, 'get_lvs', self.mock_get_lvs) + monkeypatch.setattr(process, 'call', lambda x, **kw: ('', '', 0)) + + result = migrate.get_cluster_name(osd_id='0', osd_fsid='1234') + assert "name_of_the_cluster" == result + + def test_cluster_not_found(self, monkeypatch, capsys): + self.mock_volumes = [] + self.mock_volumes.append([]) + + monkeypatch.setattr(migrate.api, 'get_lvs', self.mock_get_lvs) + monkeypatch.setattr(process, 'call', lambda x, **kw: ('', '', 0)) + + with pytest.raises(SystemExit) as error: + migrate.get_cluster_name(osd_id='0', osd_fsid='1234') + stdout, stderr = capsys.readouterr() + expected = 'Unexpected error, terminating' + assert expected in str(error.value) + expected = 'Unable to find any LV for source OSD: id:0 fsid:1234' + assert expected in stderr + +class TestFindAssociatedDevices(object): + + mock_volumes = [] + def mock_get_lvs(self, *args, **kwargs): + return self.mock_volumes.pop(0) + + mock_single_volumes = {} + def mock_get_first_lv(self, *args, **kwargs): + p = kwargs['filters']['lv_path'] + return self.mock_single_volumes[p] + + def test_lv_is_matched_id(self, monkeypatch): + tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data,ceph.osd_fsid=1234' + vol = api.Volume(lv_name='volume1', lv_uuid='y', vg_name='', + lv_path='/dev/VolGroup/lv1', lv_tags=tags) + self.mock_volumes = [] + self.mock_volumes.append([vol]) + self.mock_volumes.append([vol]) + self.mock_volumes.append([]) + self.mock_volumes.append([]) + + self.mock_single_volumes = {'/dev/VolGroup/lv1': vol} + + monkeypatch.setattr(migrate.api, 'get_lvs', self.mock_get_lvs) + monkeypatch.setattr(migrate.api, 'get_first_lv', self.mock_get_first_lv) + monkeypatch.setattr(process, 'call', lambda x, **kw: ('', '', 0)) + + result = migrate.find_associated_devices(osd_id='0', osd_fsid='1234') + assert len(result) == 1 + assert result[0][0].abspath == '/dev/VolGroup/lv1' + assert result[0][0].lvs == [vol] + assert result[0][1] == 'block' + + def test_lv_is_matched_id2(self, monkeypatch): + tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data,ceph.osd_fsid=1234' + vol = api.Volume(lv_name='volume1', lv_uuid='y', vg_name='vg', + lv_path='/dev/VolGroup/lv1', lv_tags=tags) + tags2 = 'ceph.osd_id=0,ceph.journal_uuid=xx,ceph.type=wal,ceph.osd_fsid=1234' + vol2 = api.Volume(lv_name='volume2', lv_uuid='z', vg_name='vg', + lv_path='/dev/VolGroup/lv2', lv_tags=tags2) + self.mock_volumes = [] + self.mock_volumes.append([vol]) + self.mock_volumes.append([vol]) + self.mock_volumes.append([]) + self.mock_volumes.append([vol2]) + + self.mock_single_volumes = {'/dev/VolGroup/lv1': vol, '/dev/VolGroup/lv2': vol2} + + monkeypatch.setattr(migrate.api, 'get_lvs', self.mock_get_lvs) + monkeypatch.setattr(migrate.api, 'get_first_lv', self.mock_get_first_lv) + monkeypatch.setattr(process, 'call', lambda x, **kw: ('', '', 0)) + + result = migrate.find_associated_devices(osd_id='0', osd_fsid='1234') + assert len(result) == 2 + for d in result: + if d[1] == 'block': + assert d[0].abspath == '/dev/VolGroup/lv1' + assert d[0].lvs == [vol] + elif d[1] == 'wal': + assert d[0].abspath == '/dev/VolGroup/lv2' + assert d[0].lvs == [vol2] + else: + assert False + + def test_lv_is_matched_id3(self, monkeypatch): + tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data,ceph.osd_fsid=1234' + vol = api.Volume(lv_name='volume1', lv_uuid='y', vg_name='vg', + lv_path='/dev/VolGroup/lv1', lv_tags=tags) + tags2 = 'ceph.osd_id=0,ceph.journal_uuid=xx,ceph.type=wal,ceph.osd_fsid=1234' + vol2 = api.Volume(lv_name='volume2', lv_uuid='z', vg_name='vg', + lv_path='/dev/VolGroup/lv2', lv_tags=tags2) + tags3 = 'ceph.osd_id=0,ceph.journal_uuid=xx,ceph.type=db,ceph.osd_fsid=1234' + vol3 = api.Volume(lv_name='volume3', lv_uuid='z', vg_name='vg', + lv_path='/dev/VolGroup/lv3', lv_tags=tags3) + + self.mock_volumes = [] + self.mock_volumes.append([vol]) + self.mock_volumes.append([vol]) + self.mock_volumes.append([vol3]) + self.mock_volumes.append([vol2]) + + self.mock_single_volumes = {'/dev/VolGroup/lv1': vol, + '/dev/VolGroup/lv2': vol2, + '/dev/VolGroup/lv3': vol3} + + monkeypatch.setattr(migrate.api, 'get_lvs', self.mock_get_lvs) + monkeypatch.setattr(migrate.api, 'get_first_lv', self.mock_get_first_lv) + monkeypatch.setattr(process, 'call', lambda x, **kw: ('', '', 0)) + + result = migrate.find_associated_devices(osd_id='0', osd_fsid='1234') + assert len(result) == 3 + for d in result: + if d[1] == 'block': + assert d[0].abspath == '/dev/VolGroup/lv1' + assert d[0].lvs == [vol] + elif d[1] == 'wal': + assert d[0].abspath == '/dev/VolGroup/lv2' + assert d[0].lvs == [vol2] + elif d[1] == 'db': + assert d[0].abspath == '/dev/VolGroup/lv3' + assert d[0].lvs == [vol3] + else: + assert False + + def test_lv_is_not_matched(self, monkeypatch, capsys): + self.mock_volumes = [None] + monkeypatch.setattr(migrate.api, 'get_lvs', self.mock_get_lvs) + monkeypatch.setattr(process, 'call', lambda x, **kw: ('', '', 0)) + + with pytest.raises(SystemExit) as error: + migrate.find_associated_devices(osd_id='1', osd_fsid='1234') + stdout, stderr = capsys.readouterr() + expected = 'Unexpected error, terminating' + assert expected in str(error.value) + expected = 'Unable to find any LV for source OSD: id:1 fsid:1234' + assert expected in stderr + +class TestVolumeTagTracker(object): + mock_single_volumes = {} + def mock_get_first_lv(self, *args, **kwargs): + p = kwargs['filters']['lv_path'] + return self.mock_single_volumes[p] + + mock_process_input = [] + def mock_process(self, *args, **kwargs): + self.mock_process_input.append(args[0]); + return ('', '', 0) + + def test_init(self, monkeypatch): + source_tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data,ceph.osd_fsid=1234' + source_db_tags = 'ceph.osd_id=0,journal_uuid=x,ceph.type=db, osd_fsid=1234' + source_wal_tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=wal' + target_tags="ceph.a=1,ceph.b=2,c=3,ceph.d=4" # 'c' to be bypassed + devices=[] + + data_vol = api.Volume(lv_name='volume1', lv_uuid='y', vg_name='vg', + lv_path='/dev/VolGroup/lv1', lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', lv_uuid='y', vg_name='vg', + lv_path='/dev/VolGroup/lv2', lv_tags=source_db_tags) + wal_vol = api.Volume(lv_name='volume3', lv_uuid='y', vg_name='vg', + lv_path='/dev/VolGroup/lv3', lv_tags=source_wal_tags) + + self.mock_single_volumes = {'/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol, + '/dev/VolGroup/lv3': wal_vol} + monkeypatch.setattr(migrate.api, 'get_first_lv', self.mock_get_first_lv) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + data_device = Device(path = '/dev/VolGroup/lv1') + db_device = Device(path = '/dev/VolGroup/lv2') + wal_device = Device(path = '/dev/VolGroup/lv3') + devices.append([data_device, 'block']) + devices.append([db_device, 'db']) + devices.append([wal_device, 'wal']) + + target = api.Volume(lv_name='target_name', lv_tags=target_tags, + lv_path='/dev/VolGroup/lv_target') + t = migrate.VolumeTagTracker(devices, target); + + assert 3 == len(t.old_target_tags) + + assert data_device == t.data_device + assert 4 == len(t.old_data_tags) + assert 'data' == t.old_data_tags['ceph.type'] + + assert db_device == t.db_device + assert 2 == len(t.old_db_tags) + assert 'db' == t.old_db_tags['ceph.type'] + + assert wal_device == t.wal_device + assert 3 == len(t.old_wal_tags) + assert 'wal' == t.old_wal_tags['ceph.type'] + + def test_update_tags_when_lv_create(self, monkeypatch): + source_tags = \ + 'ceph.osd_id=0,ceph.journal_uuid=x,' \ + 'ceph.type=data,ceph.osd_fsid=1234' + source_db_tags = \ + 'ceph.osd_id=0,journal_uuid=x,ceph.type=db,' \ + 'osd_fsid=1234' + + devices=[] + + data_vol = api.Volume(lv_name='volume1', lv_uuid='y', vg_name='vg', + lv_path='/dev/VolGroup/lv1', lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', lv_uuid='y', vg_name='vg', + lv_path='/dev/VolGroup/lv2', lv_tags=source_db_tags) + + self.mock_single_volumes = {'/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol} + + monkeypatch.setattr(migrate.api, 'get_first_lv', self.mock_get_first_lv) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + data_device = Device(path = '/dev/VolGroup/lv1') + db_device = Device(path = '/dev/VolGroup/lv2') + devices.append([data_device, 'block']) + devices.append([db_device, 'db']) + + target = api.Volume(lv_name='target_name', lv_tags='', + lv_uuid='wal_uuid', + lv_path='/dev/VolGroup/lv_target') + t = migrate.VolumeTagTracker(devices, target); + + self.mock_process_input = [] + t.update_tags_when_lv_create('wal') + + assert 3 == len(self.mock_process_input) + + assert ['lvchange', + '--addtag', 'ceph.wal_uuid=wal_uuid', + '--addtag', 'ceph.wal_device=/dev/VolGroup/lv_target', + '/dev/VolGroup/lv1'] == self.mock_process_input[0] + + assert self.mock_process_input[1].sort() == [ + 'lvchange', + '--addtag', 'ceph.osd_id=0', + '--addtag', 'ceph.journal_uuid=x', + '--addtag', 'ceph.type=wal', + '--addtag', 'ceph.osd_fsid=1234', + '--addtag', 'ceph.wal_uuid=wal_uuid', + '--addtag', 'ceph.wal_device=/dev/VolGroup/lv_target', + '/dev/VolGroup/lv_target'].sort() + + assert ['lvchange', + '--addtag', 'ceph.wal_uuid=wal_uuid', + '--addtag', 'ceph.wal_device=/dev/VolGroup/lv_target', + '/dev/VolGroup/lv2'] == self.mock_process_input[2] + + def test_remove_lvs(self, monkeypatch): + source_tags = \ + 'ceph.osd_id=0,ceph.journal_uuid=x,' \ + 'ceph.type=data,ceph.osd_fsid=1234,ceph.wal_uuid=aaaaa' + source_db_tags = \ + 'ceph.osd_id=0,journal_uuid=x,ceph.type=db,' \ + 'osd_fsid=1234,ceph.wal_device=aaaaa' + source_wal_tags = \ + 'ceph.wal_uuid=uuid,ceph.wal_device=device,' \ + 'ceph.osd_id=0,ceph.type=wal' + + devices=[] + + data_vol = api.Volume(lv_name='volume1', lv_uuid='y', vg_name='vg', + lv_path='/dev/VolGroup/lv1', lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', lv_uuid='y', vg_name='vg', + lv_path='/dev/VolGroup/lv2', lv_tags=source_db_tags) + wal_vol = api.Volume(lv_name='volume3', lv_uuid='y', vg_name='vg', + lv_path='/dev/VolGroup/lv3', lv_tags=source_wal_tags) + + self.mock_single_volumes = {'/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol, + '/dev/VolGroup/lv3': wal_vol} + + monkeypatch.setattr(migrate.api, 'get_first_lv', self.mock_get_first_lv) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + data_device = Device(path = '/dev/VolGroup/lv1') + db_device = Device(path = '/dev/VolGroup/lv2') + wal_device = Device(path = '/dev/VolGroup/lv3') + devices.append([data_device, 'block']) + devices.append([db_device, 'db']) + devices.append([wal_device, 'wal']) + + target = api.Volume(lv_name='target_name', lv_tags='', + lv_path='/dev/VolGroup/lv_target') + t = migrate.VolumeTagTracker(devices, target); + + device_to_remove = devices.copy() + + self.mock_process_input = [] + t.remove_lvs(device_to_remove, 'db') + + assert 3 == len(self.mock_process_input) + assert ['lvchange', + '--deltag', 'ceph.wal_uuid=uuid', + '--deltag', 'ceph.wal_device=device', + '--deltag', 'ceph.osd_id=0', + '--deltag', 'ceph.type=wal', + '/dev/VolGroup/lv3'] == self.mock_process_input[0] + assert ['lvchange', + '--deltag', 'ceph.wal_uuid=aaaaa', + '/dev/VolGroup/lv1'] == self.mock_process_input[1] + assert ['lvchange', + '--deltag', 'ceph.wal_device=aaaaa', + '/dev/VolGroup/lv2'] == self.mock_process_input[2] + + def test_replace_lvs(self, monkeypatch): + source_tags = \ + 'ceph.osd_id=0,ceph.type=data,ceph.osd_fsid=1234,'\ + 'ceph.wal_uuid=wal_uuid,ceph.db_device=/dbdevice' + source_db_tags = \ + 'ceph.osd_id=0,ceph.type=db,ceph.osd_fsid=1234' + source_wal_tags = \ + 'ceph.wal_uuid=uuid,ceph.wal_device=device,' \ + 'ceph.osd_id=0,ceph.type=wal' + + devices=[] + + data_vol = api.Volume(lv_name='volume1', lv_uuid='datauuid', vg_name='vg', + lv_path='/dev/VolGroup/lv1', lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', lv_uuid='dbuuid', vg_name='vg', + lv_path='/dev/VolGroup/lv2', lv_tags=source_db_tags) + wal_vol = api.Volume(lv_name='volume3', lv_uuid='waluuid', vg_name='vg', + lv_path='/dev/VolGroup/lv3', lv_tags=source_wal_tags) + + self.mock_single_volumes = {'/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol, + '/dev/VolGroup/lv3': wal_vol} + + monkeypatch.setattr(migrate.api, 'get_first_lv', self.mock_get_first_lv) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + data_device = Device(path = '/dev/VolGroup/lv1') + db_device = Device(path = '/dev/VolGroup/lv2') + wal_device = Device(path = '/dev/VolGroup/lv3') + devices.append([data_device, 'block']) + devices.append([db_device, 'db']) + devices.append([wal_device, 'wal']) + + target = api.Volume(lv_name='target_name', + lv_uuid='ttt', + lv_tags='ceph.tag_to_remove=aaa', + lv_path='/dev/VolGroup/lv_target') + t = migrate.VolumeTagTracker(devices, target); + + self.mock_process_input = [] + t.replace_lvs(devices, 'db') + + assert 5 == len(self.mock_process_input) + + assert ['lvchange', + '--deltag', 'ceph.osd_id=0', + '--deltag', 'ceph.type=db', + '--deltag', 'ceph.osd_fsid=1234', + '/dev/VolGroup/lv2'] == self.mock_process_input[0] + assert ['lvchange', + '--deltag', 'ceph.wal_uuid=uuid', + '--deltag', 'ceph.wal_device=device', + '--deltag', 'ceph.osd_id=0', + '--deltag', 'ceph.type=wal', + '/dev/VolGroup/lv3'] == self.mock_process_input[1] + assert ['lvchange', + '--deltag', 'ceph.db_device=/dbdevice', + '--deltag', 'ceph.wal_uuid=wal_uuid', + '/dev/VolGroup/lv1'] == self.mock_process_input[2] + + assert ['lvchange', + '--addtag', 'ceph.db_uuid=ttt', + '--addtag', 'ceph.db_device=/dev/VolGroup/lv_target', + '/dev/VolGroup/lv1'] == self.mock_process_input[3] + + assert self.mock_process_input[4].sort() == [ + 'lvchange', + '--addtag', 'ceph.osd_id=0', + '--addtag', 'ceph.osd_fsid=1234', + '--addtag', 'ceph.type=db', + '--addtag', 'ceph.db_uuid=ttt', + '--addtag', 'ceph.db_device=/dev/VolGroup/lv_target', + '/dev/VolGroup/lv_target'].sort() + + def test_undo(self, monkeypatch): + source_tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data,ceph.osd_fsid=1234' + source_db_tags = 'ceph.osd_id=0,journal_uuid=x,ceph.type=db, osd_fsid=1234' + source_wal_tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=wal' + target_tags="" + devices=[] + + data_vol = api.Volume(lv_name='volume1', lv_uuid='y', vg_name='vg', + lv_path='/dev/VolGroup/lv1', lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', lv_uuid='y', vg_name='vg', + lv_path='/dev/VolGroup/lv2', lv_tags=source_db_tags) + wal_vol = api.Volume(lv_name='volume3', lv_uuid='y', vg_name='vg', + lv_path='/dev/VolGroup/lv3', lv_tags=source_wal_tags) + + self.mock_single_volumes = {'/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol, + '/dev/VolGroup/lv3': wal_vol} + + monkeypatch.setattr(migrate.api, 'get_first_lv', self.mock_get_first_lv) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + data_device = Device(path = '/dev/VolGroup/lv1') + db_device = Device(path = '/dev/VolGroup/lv2') + wal_device = Device(path = '/dev/VolGroup/lv3') + devices.append([data_device, 'block']) + devices.append([db_device, 'db']) + devices.append([wal_device, 'wal']) + + target = api.Volume(lv_name='target_name', lv_tags=target_tags, + lv_path='/dev/VolGroup/lv_target') + t = migrate.VolumeTagTracker(devices, target); + + target.tags['ceph.a'] = 'aa'; + target.tags['ceph.b'] = 'bb'; + + data_vol.tags['ceph.journal_uuid'] = 'z'; + + db_vol.tags.pop('ceph.type') + + wal_vol.tags.clear() + + assert 2 == len(target.tags) + assert 4 == len(data_vol.tags) + assert 1 == len(db_vol.tags) + + self.mock_process_input = [] + t.undo() + + assert 0 == len(target.tags) + assert 4 == len(data_vol.tags) + assert 'x' == data_vol.tags['ceph.journal_uuid'] + + assert 2 == len(db_vol.tags) + assert 'db' == db_vol.tags['ceph.type'] + + assert 3 == len(wal_vol.tags) + assert 'wal' == wal_vol.tags['ceph.type'] + + assert 6 == len(self.mock_process_input) + assert 'lvchange' in self.mock_process_input[0] + assert '--deltag' in self.mock_process_input[0] + assert 'ceph.journal_uuid=z' in self.mock_process_input[0] + assert '/dev/VolGroup/lv1' in self.mock_process_input[0] + + assert 'lvchange' in self.mock_process_input[1] + assert '--addtag' in self.mock_process_input[1] + assert 'ceph.journal_uuid=x' in self.mock_process_input[1] + assert '/dev/VolGroup/lv1' in self.mock_process_input[1] + + assert 'lvchange' in self.mock_process_input[2] + assert '--deltag' in self.mock_process_input[2] + assert 'ceph.osd_id=0' in self.mock_process_input[2] + assert '/dev/VolGroup/lv2' in self.mock_process_input[2] + + assert 'lvchange' in self.mock_process_input[3] + assert '--addtag' in self.mock_process_input[3] + assert 'ceph.type=db' in self.mock_process_input[3] + assert '/dev/VolGroup/lv2' in self.mock_process_input[3] + + assert 'lvchange' in self.mock_process_input[4] + assert '--addtag' in self.mock_process_input[4] + assert 'ceph.type=wal' in self.mock_process_input[4] + assert '/dev/VolGroup/lv3' in self.mock_process_input[4] + + assert 'lvchange' in self.mock_process_input[5] + assert '--deltag' in self.mock_process_input[5] + assert 'ceph.a=aa' in self.mock_process_input[5] + assert 'ceph.b=bb' in self.mock_process_input[5] + assert '/dev/VolGroup/lv_target' in self.mock_process_input[5] + +class TestNew(object): + + mock_volume = None + def mock_get_lv_by_fullname(self, *args, **kwargs): + return self.mock_volume + + mock_process_input = [] + def mock_process(self, *args, **kwargs): + self.mock_process_input.append(args[0]); + return ('', '', 0) + + mock_single_volumes = {} + def mock_get_first_lv(self, *args, **kwargs): + p = kwargs['filters']['lv_path'] + return self.mock_single_volumes[p] + + mock_volumes = [] + def mock_get_lvs(self, *args, **kwargs): + return self.mock_volumes.pop(0) + + def test_newdb_non_root(self): + with pytest.raises(Exception) as error: + migrate.NewDB(argv=[ + '--osd-id', '1', + '--osd-fsid', '55BD4219-16A7-4037-BC20-0F158EFCC83D', + '--target', 'vgname/new_db']).main() + expected = 'This command needs to be executed with sudo or as root' + assert expected in str(error.value) + + @patch('os.getuid') + def test_newdb_not_target_lvm(self, m_getuid, capsys): + m_getuid.return_value = 0 + with pytest.raises(SystemExit) as error: + migrate.NewDB(argv=[ + '--osd-id', '1', + '--osd-fsid', '55BD4219-16A7-4037-BC20-0F158EFCC83D', + '--target', 'vgname/new_db']).main() + stdout, stderr = capsys.readouterr() + expected = 'Unable to attach new volume : vgname/new_db' + assert expected in str(error.value) + expected = 'Target path vgname/new_db is not a Logical Volume' + assert expected in stderr + + + @patch('os.getuid') + def test_newdb_already_in_use(self, m_getuid, monkeypatch, capsys): + m_getuid.return_value = 0 + + self.mock_volume = api.Volume(lv_name='volume1', + lv_uuid='y', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags='ceph.osd_id=5') # this results in set used_by_ceph + monkeypatch.setattr(api, 'get_lv_by_fullname', self.mock_get_lv_by_fullname) + + with pytest.raises(SystemExit) as error: + migrate.NewDB(argv=[ + '--osd-id', '1', + '--osd-fsid', '55BD4219-16A7-4037-BC20-0F158EFCC83D', + '--target', 'vgname/new_db']).main() + stdout, stderr = capsys.readouterr() + expected = 'Unable to attach new volume : vgname/new_db' + assert expected in str(error.value) + expected = 'Target Logical Volume is already used by ceph: vgname/new_db' + assert expected in stderr + + @patch('os.getuid') + def test_newdb(self, m_getuid, monkeypatch, capsys): + m_getuid.return_value = 0 + + source_tags = \ + 'ceph.osd_id=0,ceph.type=data,ceph.osd_fsid=1234,'\ + 'ceph.wal_uuid=wal_uuid,ceph.db_device=/dbdevice' + source_wal_tags = \ + 'ceph.wal_uuid=uuid,ceph.wal_device=device,' \ + 'ceph.osd_id=0,ceph.type=wal' + + data_vol = api.Volume(lv_name='volume1', lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags=source_tags) + wal_vol = api.Volume(lv_name='volume3', + lv_uuid='waluuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv3', + lv_tags=source_wal_tags) + + self.mock_single_volumes = {'/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv3': wal_vol} + + monkeypatch.setattr(migrate.api, 'get_first_lv', + self.mock_get_first_lv) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + self.mock_volume = api.Volume(lv_name='target_volume1', lv_uuid='y', + vg_name='vg', + lv_path='/dev/VolGroup/target_volume', + lv_tags='') + monkeypatch.setattr(api, 'get_lv_by_fullname', + self.mock_get_lv_by_fullname) + + monkeypatch.setattr("ceph_volume.systemd.systemctl.osd_is_active", + lambda id: False) + + #find_associated_devices will call get_lvs() 4 times + # and it this needs results to be arranged that way + self.mock_volumes = [] + self.mock_volumes.append([data_vol, wal_vol]) + self.mock_volumes.append([data_vol]) + self.mock_volumes.append([]) + self.mock_volumes.append([wal_vol]) + + monkeypatch.setattr(migrate.api, 'get_lvs', self.mock_get_lvs) + + monkeypatch.setattr(migrate, 'get_cluster_name', + lambda osd_id, osd_fsid: 'ceph_cluster') + monkeypatch.setattr(system, 'chown', lambda path: 0) + + migrate.NewDB(argv=[ + '--osd-id', '1', + '--osd-fsid', '55BD4219-16A7-4037-BC20-0F158EFCC83D', + '--target', 'vgname/new_db']).main() + + n = len(self.mock_process_input) + assert n >= 5 + + assert self.mock_process_input[n - 5] == [ + 'lvchange', + '--deltag', 'ceph.db_device=/dbdevice', + '/dev/VolGroup/lv1'] + assert self.mock_process_input[n - 4] == [ + 'lvchange', + '--addtag', 'ceph.db_uuid=y', + '--addtag', 'ceph.db_device=/dev/VolGroup/target_volume', + '/dev/VolGroup/lv1'] + + assert self.mock_process_input[n - 3].sort() == [ + 'lvchange', + '--addtag', 'ceph.wal_uuid=uuid', + '--addtag', 'ceph.osd_id=0', + '--addtag', 'ceph.type=db', + '--addtag', 'ceph.osd_fsid=1234', + '--addtag', 'ceph.db_uuid=y', + '--addtag', 'ceph.db_device=/dev/VolGroup/target_volume', + '/dev/VolGroup/target_volume'].sort() + + assert self.mock_process_input[n - 2] == [ + 'lvchange', + '--addtag', 'ceph.db_uuid=y', + '--addtag', 'ceph.db_device=/dev/VolGroup/target_volume', + '/dev/VolGroup/lv3'] + + assert self.mock_process_input[n - 1] == [ + 'ceph-bluestore-tool', + '--path', '/var/lib/ceph/osd/ceph_cluster-1', + '--dev-target', '/dev/VolGroup/target_volume', + '--command', 'bluefs-bdev-new-db'] + + def test_newdb_active_systemd(self, is_root, monkeypatch, capsys): + source_tags = \ + 'ceph.osd_id=0,ceph.type=data,ceph.osd_fsid=1234,'\ + 'ceph.wal_uuid=wal_uuid,ceph.db_device=/dbdevice' + source_wal_tags = \ + 'ceph.wal_uuid=uuid,ceph.wal_device=device,' \ + 'ceph.osd_id=0,ceph.type=wal' + + data_vol = api.Volume(lv_name='volume1', lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags=source_tags) + wal_vol = api.Volume(lv_name='volume3', + lv_uuid='waluuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv3', + lv_tags=source_wal_tags) + + self.mock_single_volumes = {'/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv3': wal_vol} + + monkeypatch.setattr(migrate.api, 'get_first_lv', + self.mock_get_first_lv) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + self.mock_volume = api.Volume(lv_name='target_volume1', lv_uuid='y', + vg_name='vg', + lv_path='/dev/VolGroup/target_volume', + lv_tags='') + monkeypatch.setattr(api, 'get_lv_by_fullname', + self.mock_get_lv_by_fullname) + + monkeypatch.setattr("ceph_volume.systemd.systemctl.osd_is_active", + lambda id: True) + + #find_associated_devices will call get_lvs() 4 times + # and it this needs results to be arranged that way + self.mock_volumes = [] + self.mock_volumes.append([data_vol, wal_vol]) + self.mock_volumes.append([data_vol]) + self.mock_volumes.append([]) + self.mock_volumes.append([wal_vol]) + + monkeypatch.setattr(migrate.api, 'get_lvs', self.mock_get_lvs) + + monkeypatch.setattr(migrate, 'get_cluster_name', + lambda osd_id, osd_fsid: 'ceph_cluster') + monkeypatch.setattr(system, 'chown', lambda path: 0) + + m = migrate.NewDB(argv=[ + '--osd-id', '1', + '--osd-fsid', '55BD4219-16A7-4037-BC20-0F158EFCC83D', + '--target', 'vgname/new_db']) + + with pytest.raises(SystemExit) as error: + m.main() + + stdout, stderr = capsys.readouterr() + + assert 'Unable to attach new volume for OSD: 1' == str(error.value) + assert '--> OSD ID is running, stop it with: systemctl stop ceph-osd@1' == stderr.rstrip() + assert not stdout + + def test_newdb_no_systemd(self, is_root, monkeypatch): + source_tags = \ + 'ceph.osd_id=0,ceph.type=data,ceph.osd_fsid=1234,'\ + 'ceph.wal_uuid=wal_uuid,ceph.db_device=/dbdevice' + source_wal_tags = \ + 'ceph.wal_uuid=uuid,ceph.wal_device=device,' \ + 'ceph.osd_id=0,ceph.type=wal' + + data_vol = api.Volume(lv_name='volume1', lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags=source_tags) + wal_vol = api.Volume(lv_name='volume3', + lv_uuid='waluuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv3', + lv_tags=source_wal_tags) + + self.mock_single_volumes = {'/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv3': wal_vol} + + monkeypatch.setattr(migrate.api, 'get_first_lv', + self.mock_get_first_lv) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + self.mock_volume = api.Volume(lv_name='target_volume1', lv_uuid='y', + vg_name='vg', + lv_path='/dev/VolGroup/target_volume', + lv_tags='') + monkeypatch.setattr(api, 'get_lv_by_fullname', + self.mock_get_lv_by_fullname) + + #find_associated_devices will call get_lvs() 4 times + # and it this needs results to be arranged that way + self.mock_volumes = [] + self.mock_volumes.append([data_vol, wal_vol]) + self.mock_volumes.append([data_vol]) + self.mock_volumes.append([]) + self.mock_volumes.append([wal_vol]) + + monkeypatch.setattr(migrate.api, 'get_lvs', self.mock_get_lvs) + + monkeypatch.setattr(migrate, 'get_cluster_name', + lambda osd_id, osd_fsid: 'ceph_cluster') + monkeypatch.setattr(system, 'chown', lambda path: 0) + + migrate.NewDB(argv=[ + '--osd-id', '1', + '--osd-fsid', '55BD4219-16A7-4037-BC20-0F158EFCC83D', + '--target', 'vgname/new_db', + '--no-systemd']).main() + + n = len(self.mock_process_input) + assert n >= 5 + + assert self.mock_process_input[n - 5] == [ + 'lvchange', + '--deltag', 'ceph.db_device=/dbdevice', + '/dev/VolGroup/lv1'] + assert self.mock_process_input[n - 4] == [ + 'lvchange', + '--addtag', 'ceph.db_uuid=y', + '--addtag', 'ceph.db_device=/dev/VolGroup/target_volume', + '/dev/VolGroup/lv1'] + + assert self.mock_process_input[n - 3].sort() == [ + 'lvchange', + '--addtag', 'ceph.wal_uuid=uuid', + '--addtag', 'ceph.osd_id=0', + '--addtag', 'ceph.type=db', + '--addtag', 'ceph.osd_fsid=1234', + '--addtag', 'ceph.db_uuid=y', + '--addtag', 'ceph.db_device=/dev/VolGroup/target_volume', + '/dev/VolGroup/target_volume'].sort() + + assert self.mock_process_input[n - 2] == [ + 'lvchange', + '--addtag', 'ceph.db_uuid=y', + '--addtag', 'ceph.db_device=/dev/VolGroup/target_volume', + '/dev/VolGroup/lv3'] + + assert self.mock_process_input[n - 1] == [ + 'ceph-bluestore-tool', + '--path', '/var/lib/ceph/osd/ceph_cluster-1', + '--dev-target', '/dev/VolGroup/target_volume', + '--command', 'bluefs-bdev-new-db'] + + @patch('os.getuid') + def test_newwal(self, m_getuid, monkeypatch, capsys): + m_getuid.return_value = 0 + + source_tags = \ + 'ceph.osd_id=0,ceph.type=data,ceph.osd_fsid=1234' + + data_vol = api.Volume(lv_name='volume1', lv_uuid='datauuid', vg_name='vg', + lv_path='/dev/VolGroup/lv1', lv_tags=source_tags) + + self.mock_single_volumes = {'/dev/VolGroup/lv1': data_vol} + + monkeypatch.setattr(migrate.api, 'get_first_lv', self.mock_get_first_lv) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + self.mock_volume = api.Volume(lv_name='target_volume1', lv_uuid='y', vg_name='vg', + lv_path='/dev/VolGroup/target_volume', + lv_tags='') + monkeypatch.setattr(api, 'get_lv_by_fullname', self.mock_get_lv_by_fullname) + + monkeypatch.setattr("ceph_volume.systemd.systemctl.osd_is_active", lambda id: False) + + #find_associated_devices will call get_lvs() 4 times + # and it this needs results to be arranged that way + self.mock_volumes = [] + self.mock_volumes.append([data_vol]) + self.mock_volumes.append([data_vol]) + self.mock_volumes.append([]) + self.mock_volumes.append([]) + + monkeypatch.setattr(migrate.api, 'get_lvs', self.mock_get_lvs) + + monkeypatch.setattr(migrate, 'get_cluster_name', lambda osd_id, osd_fsid: 'cluster') + monkeypatch.setattr(system, 'chown', lambda path: 0) + + migrate.NewWAL(argv=[ + '--osd-id', '2', + '--osd-fsid', '55BD4219-16A7-4037-BC20-0F158EFCC83D', + '--target', 'vgname/new_wal']).main() + + n = len(self.mock_process_input) + assert n >= 3 + + assert self.mock_process_input[n - 3] == [ + 'lvchange', + '--addtag', 'ceph.wal_uuid=y', + '--addtag', 'ceph.wal_device=/dev/VolGroup/target_volume', + '/dev/VolGroup/lv1'] + + assert self.mock_process_input[n - 2].sort() == [ + 'lvchange', + '--addtag', 'ceph.osd_id=0', + '--addtag', 'ceph.type=wal', + '--addtag', 'ceph.osd_fsid=1234', + '--addtag', 'ceph.wal_uuid=y', + '--addtag', 'ceph.wal_device=/dev/VolGroup/target_volume', + '/dev/VolGroup/target_volume'].sort() + + assert self.mock_process_input[n - 1] == [ + 'ceph-bluestore-tool', + '--path', '/var/lib/ceph/osd/cluster-2', + '--dev-target', '/dev/VolGroup/target_volume', + '--command', 'bluefs-bdev-new-wal'] + + def test_newwal_active_systemd(self, is_root, monkeypatch, capsys): + source_tags = \ + 'ceph.osd_id=0,ceph.type=data,ceph.osd_fsid=1234' + + data_vol = api.Volume(lv_name='volume1', lv_uuid='datauuid', vg_name='vg', + lv_path='/dev/VolGroup/lv1', lv_tags=source_tags) + + self.mock_single_volumes = {'/dev/VolGroup/lv1': data_vol} + + monkeypatch.setattr(migrate.api, 'get_first_lv', self.mock_get_first_lv) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + self.mock_volume = api.Volume(lv_name='target_volume1', lv_uuid='y', vg_name='vg', + lv_path='/dev/VolGroup/target_volume', + lv_tags='') + monkeypatch.setattr(api, 'get_lv_by_fullname', self.mock_get_lv_by_fullname) + + monkeypatch.setattr("ceph_volume.systemd.systemctl.osd_is_active", lambda id: True) + + #find_associated_devices will call get_lvs() 4 times + # and it this needs results to be arranged that way + self.mock_volumes = [] + self.mock_volumes.append([data_vol]) + self.mock_volumes.append([data_vol]) + self.mock_volumes.append([]) + self.mock_volumes.append([]) + + monkeypatch.setattr(migrate.api, 'get_lvs', self.mock_get_lvs) + + monkeypatch.setattr(migrate, 'get_cluster_name', lambda osd_id, osd_fsid: 'cluster') + monkeypatch.setattr(system, 'chown', lambda path: 0) + + m = migrate.NewWAL(argv=[ + '--osd-id', '2', + '--osd-fsid', '55BD4219-16A7-4037-BC20-0F158EFCC83D', + '--target', 'vgname/new_wal']) + + with pytest.raises(SystemExit) as error: + m.main() + + stdout, stderr = capsys.readouterr() + + assert 'Unable to attach new volume for OSD: 2' == str(error.value) + assert '--> OSD ID is running, stop it with: systemctl stop ceph-osd@2' == stderr.rstrip() + assert not stdout + + def test_newwal_no_systemd(self, is_root, monkeypatch): + source_tags = \ + 'ceph.osd_id=0,ceph.type=data,ceph.osd_fsid=1234' + + data_vol = api.Volume(lv_name='volume1', lv_uuid='datauuid', vg_name='vg', + lv_path='/dev/VolGroup/lv1', lv_tags=source_tags) + + self.mock_single_volumes = {'/dev/VolGroup/lv1': data_vol} + + monkeypatch.setattr(migrate.api, 'get_first_lv', self.mock_get_first_lv) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + self.mock_volume = api.Volume(lv_name='target_volume1', lv_uuid='y', vg_name='vg', + lv_path='/dev/VolGroup/target_volume', + lv_tags='') + monkeypatch.setattr(api, 'get_lv_by_fullname', self.mock_get_lv_by_fullname) + + #find_associated_devices will call get_lvs() 4 times + # and it this needs results to be arranged that way + self.mock_volumes = [] + self.mock_volumes.append([data_vol]) + self.mock_volumes.append([data_vol]) + self.mock_volumes.append([]) + self.mock_volumes.append([]) + + monkeypatch.setattr(migrate.api, 'get_lvs', self.mock_get_lvs) + + monkeypatch.setattr(migrate, 'get_cluster_name', lambda osd_id, osd_fsid: 'cluster') + monkeypatch.setattr(system, 'chown', lambda path: 0) + + migrate.NewWAL(argv=[ + '--osd-id', '2', + '--osd-fsid', '55BD4219-16A7-4037-BC20-0F158EFCC83D', + '--target', 'vgname/new_wal', + '--no-systemd']).main() + + n = len(self.mock_process_input) + assert n >= 3 + + assert self.mock_process_input[n - 3] == [ + 'lvchange', + '--addtag', 'ceph.wal_uuid=y', + '--addtag', 'ceph.wal_device=/dev/VolGroup/target_volume', + '/dev/VolGroup/lv1'] + + assert self.mock_process_input[n - 2].sort() == [ + 'lvchange', + '--addtag', 'ceph.osd_id=0', + '--addtag', 'ceph.type=wal', + '--addtag', 'ceph.osd_fsid=1234', + '--addtag', 'ceph.wal_uuid=y', + '--addtag', 'ceph.wal_device=/dev/VolGroup/target_volume', + '/dev/VolGroup/target_volume'].sort() + + assert self.mock_process_input[n - 1] == [ + 'ceph-bluestore-tool', + '--path', '/var/lib/ceph/osd/cluster-2', + '--dev-target', '/dev/VolGroup/target_volume', + '--command', 'bluefs-bdev-new-wal'] + +class TestMigrate(object): + + mock_volume = None + def mock_get_lv_by_fullname(self, *args, **kwargs): + return self.mock_volume + + mock_process_input = [] + def mock_process(self, *args, **kwargs): + self.mock_process_input.append(args[0]); + return ('', '', 0) + + mock_single_volumes = {} + def mock_get_first_lv(self, *args, **kwargs): + p = kwargs['filters']['lv_path'] + return self.mock_single_volumes[p] + + mock_volumes = [] + def mock_get_lvs(self, *args, **kwargs): + return self.mock_volumes.pop(0) + + def test_get_source_devices(self, monkeypatch): + + source_tags = 'ceph.osd_id=2,ceph.type=data,ceph.osd_fsid=1234' + source_db_tags = 'ceph.osd_id=2,ceph.type=db,ceph.osd_fsid=1234' + source_wal_tags = 'ceph.osd_id=2,ceph.type=wal,ceph.osd_fsid=1234' + + data_vol = api.Volume(lv_name='volume1', + lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', + lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2', + lv_tags=source_db_tags) + + wal_vol = api.Volume(lv_name='volume3', + lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv3', + lv_tags=source_wal_tags) + + self.mock_single_volumes = { + '/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol, + '/dev/VolGroup/lv3': wal_vol, + } + monkeypatch.setattr(migrate.api, 'get_first_lv', + self.mock_get_first_lv) + + self.mock_volume = api.Volume(lv_name='volume2', lv_uuid='y', + vg_name='vg', + lv_path='/dev/VolGroup/lv2', + lv_tags='ceph.osd_id=5,ceph.osd_type=db') + monkeypatch.setattr(api, 'get_lv_by_fullname', + self.mock_get_lv_by_fullname) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + devices = [] + devices.append([Device('/dev/VolGroup/lv1'), 'block']) + devices.append([Device('/dev/VolGroup/lv2'), 'db']) + devices.append([Device('/dev/VolGroup/lv3'), 'wal']) + + monkeypatch.setattr(migrate, 'find_associated_devices', + lambda osd_id, osd_fsid: devices) + + + argv = [ + '--osd-id', '2', + '--osd-fsid', '55BD4219-16A7-4037-BC20-0F158EFCC83D', + '--from', 'data', 'wal', + '--target', 'vgname/new_wal' + ] + m = migrate.Migrate(argv=argv) + m.args = m.make_parser('ceph-volume lvm migation', 'help').parse_args(argv) + res_devices = m.get_source_devices(devices) + + assert 2 == len(res_devices) + assert devices[0] == res_devices[0] + assert devices[2] == res_devices[1] + + argv = [ + '--osd-id', '2', + '--osd-fsid', '55BD4219-16A7-4037-BC20-0F158EFCC83D', + '--from', 'db', 'wal', 'data', + '--target', 'vgname/new_wal' + ] + m = migrate.Migrate(argv=argv) + m.args = m.make_parser('ceph-volume lvm migation', 'help').parse_args(argv) + res_devices = m.get_source_devices(devices) + + assert 3 == len(res_devices) + assert devices[0] == res_devices[0] + assert devices[1] == res_devices[1] + assert devices[2] == res_devices[2] + + + def test_migrate_without_args(self, capsys): + help_msg = """ +Moves BlueFS data from source volume(s) to the target one, source +volumes (except the main (i.e. data or block) one) are removed on +success. LVM volumes are permitted for Target only, both already +attached or new logical one. In the latter case it is attached to OSD +replacing one of the source devices. Following replacement rules apply +(in the order of precedence, stop on the first match): +* if source list has DB volume - target device replaces it. +* if source list has WAL volume - target device replace it. +* if source list has slow volume only - operation is not permitted, + requires explicit allocation via new-db/new-wal command. + +Example calls for supported scenarios: + + Moves BlueFS data from main device to LV already attached as DB: + + ceph-volume lvm migrate --osd-id 1 --osd-fsid --from data --target vgname/db + + Moves BlueFS data from shared main device to LV which will be attached + as a new DB: + + ceph-volume lvm migrate --osd-id 1 --osd-fsid --from data --target vgname/new_db + + Moves BlueFS data from DB device to new LV, DB is replaced: + + ceph-volume lvm migrate --osd-id 1 --osd-fsid --from db --target vgname/new_db + + Moves BlueFS data from main and DB devices to new LV, DB is replaced: + + ceph-volume lvm migrate --osd-id 1 --osd-fsid --from data db --target vgname/new_db + + Moves BlueFS data from main, DB and WAL devices to new LV, WAL is + removed and DB is replaced: + + ceph-volume lvm migrate --osd-id 1 --osd-fsid --from data db wal --target vgname/new_db + + Moves BlueFS data from main, DB and WAL devices to main device, WAL + and DB are removed: + + ceph-volume lvm migrate --osd-id 1 --osd-fsid --from db wal --target vgname/data + +""" + m = migrate.Migrate(argv=[]) + m.main() + stdout, stderr = capsys.readouterr() + assert help_msg in stdout + assert not stderr + + + @patch('os.getuid') + def test_migrate_data_db_to_new_db(self, m_getuid, monkeypatch): + m_getuid.return_value = 0 + + source_tags = 'ceph.osd_id=2,ceph.type=data,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev' + source_db_tags = 'ceph.osd_id=2,ceph.type=db,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev' + + data_vol = api.Volume(lv_name='volume1', + lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', + lv_uuid='dbuuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2', + lv_tags=source_db_tags) + + self.mock_single_volumes = { + '/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol, + } + monkeypatch.setattr(migrate.api, 'get_first_lv', + self.mock_get_first_lv) + + self.mock_volume = api.Volume(lv_name='volume2_new', lv_uuid='new-db-uuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2_new', + lv_tags='') + monkeypatch.setattr(api, 'get_lv_by_fullname', + self.mock_get_lv_by_fullname) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + devices = [] + devices.append([Device('/dev/VolGroup/lv1'), 'block']) + devices.append([Device('/dev/VolGroup/lv2'), 'db']) + + monkeypatch.setattr(migrate, 'find_associated_devices', + lambda osd_id, osd_fsid: devices) + + + monkeypatch.setattr("ceph_volume.systemd.systemctl.osd_is_active", + lambda id: False) + + monkeypatch.setattr(migrate, 'get_cluster_name', + lambda osd_id, osd_fsid: 'ceph') + monkeypatch.setattr(system, 'chown', lambda path: 0) + m = migrate.Migrate(argv=[ + '--osd-id', '2', + '--osd-fsid', '1234', + '--from', 'data', 'db', 'wal', + '--target', 'vgname/new_wal']) + m.main() + + n = len(self.mock_process_input) + assert n >= 5 + + assert self. mock_process_input[n-5] == [ + 'lvchange', + '--deltag', 'ceph.osd_id=2', + '--deltag', 'ceph.type=db', + '--deltag', 'ceph.osd_fsid=1234', + '--deltag', 'ceph.cluster_name=ceph', + '--deltag', 'ceph.db_uuid=dbuuid', + '--deltag', 'ceph.db_device=db_dev', + '/dev/VolGroup/lv2'] + + assert self. mock_process_input[n-4] == [ + 'lvchange', + '--deltag', 'ceph.db_uuid=dbuuid', + '--deltag', 'ceph.db_device=db_dev', + '/dev/VolGroup/lv1'] + + assert self. mock_process_input[n-3] == [ + 'lvchange', + '--addtag', 'ceph.db_uuid=new-db-uuid', + '--addtag', 'ceph.db_device=/dev/VolGroup/lv2_new', + '/dev/VolGroup/lv1'] + + assert self. mock_process_input[n-2] == [ + 'lvchange', + '--addtag', 'ceph.osd_id=2', + '--addtag', 'ceph.type=db', + '--addtag', 'ceph.osd_fsid=1234', + '--addtag', 'ceph.cluster_name=ceph', + '--addtag', 'ceph.db_uuid=new-db-uuid', + '--addtag', 'ceph.db_device=/dev/VolGroup/lv2_new', + '/dev/VolGroup/lv2_new'] + + assert self. mock_process_input[n-1] == [ + 'ceph-bluestore-tool', + '--path', '/var/lib/ceph/osd/ceph-2', + '--dev-target', '/dev/VolGroup/lv2_new', + '--command', 'bluefs-bdev-migrate', + '--devs-source', '/var/lib/ceph/osd/ceph-2/block', + '--devs-source', '/var/lib/ceph/osd/ceph-2/block.db'] + + def test_migrate_data_db_to_new_db_active_systemd(self, is_root, monkeypatch, capsys): + source_tags = 'ceph.osd_id=2,ceph.type=data,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev' + source_db_tags = 'ceph.osd_id=2,ceph.type=db,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev' + + data_vol = api.Volume(lv_name='volume1', + lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', + lv_uuid='dbuuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2', + lv_tags=source_db_tags) + + self.mock_single_volumes = { + '/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol, + } + monkeypatch.setattr(migrate.api, 'get_first_lv', + self.mock_get_first_lv) + + self.mock_volume = api.Volume(lv_name='volume2_new', lv_uuid='new-db-uuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2_new', + lv_tags='') + monkeypatch.setattr(api, 'get_lv_by_fullname', + self.mock_get_lv_by_fullname) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + devices = [] + devices.append([Device('/dev/VolGroup/lv1'), 'block']) + devices.append([Device('/dev/VolGroup/lv2'), 'db']) + + monkeypatch.setattr(migrate, 'find_associated_devices', + lambda osd_id, osd_fsid: devices) + + + monkeypatch.setattr("ceph_volume.systemd.systemctl.osd_is_active", + lambda id: True) + + monkeypatch.setattr(migrate, 'get_cluster_name', + lambda osd_id, osd_fsid: 'ceph') + monkeypatch.setattr(system, 'chown', lambda path: 0) + m = migrate.Migrate(argv=[ + '--osd-id', '2', + '--osd-fsid', '1234', + '--from', 'data', 'db', 'wal', + '--target', 'vgname/new_wal']) + + with pytest.raises(SystemExit) as error: + m.main() + + stdout, stderr = capsys.readouterr() + + assert 'Unable to migrate devices associated with OSD ID: 2' == str(error.value) + assert '--> OSD is running, stop it with: systemctl stop ceph-osd@2' == stderr.rstrip() + assert not stdout + + def test_migrate_data_db_to_new_db_no_systemd(self, is_root, monkeypatch): + source_tags = 'ceph.osd_id=2,ceph.type=data,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev' + source_db_tags = 'ceph.osd_id=2,ceph.type=db,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev' + + data_vol = api.Volume(lv_name='volume1', + lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', + lv_uuid='dbuuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2', + lv_tags=source_db_tags) + + self.mock_single_volumes = { + '/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol, + } + monkeypatch.setattr(migrate.api, 'get_first_lv', + self.mock_get_first_lv) + + self.mock_volume = api.Volume(lv_name='volume2_new', lv_uuid='new-db-uuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2_new', + lv_tags='') + monkeypatch.setattr(api, 'get_lv_by_fullname', + self.mock_get_lv_by_fullname) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + devices = [] + devices.append([Device('/dev/VolGroup/lv1'), 'block']) + devices.append([Device('/dev/VolGroup/lv2'), 'db']) + + monkeypatch.setattr(migrate, 'find_associated_devices', + lambda osd_id, osd_fsid: devices) + + + monkeypatch.setattr(migrate, 'get_cluster_name', + lambda osd_id, osd_fsid: 'ceph') + monkeypatch.setattr(system, 'chown', lambda path: 0) + m = migrate.Migrate(argv=[ + '--osd-id', '2', + '--osd-fsid', '1234', + '--from', 'data', 'db', 'wal', + '--target', 'vgname/new_wal', + '--no-systemd']) + m.main() + + n = len(self.mock_process_input) + assert n >= 5 + + assert self. mock_process_input[n-5] == [ + 'lvchange', + '--deltag', 'ceph.osd_id=2', + '--deltag', 'ceph.type=db', + '--deltag', 'ceph.osd_fsid=1234', + '--deltag', 'ceph.cluster_name=ceph', + '--deltag', 'ceph.db_uuid=dbuuid', + '--deltag', 'ceph.db_device=db_dev', + '/dev/VolGroup/lv2'] + + assert self. mock_process_input[n-4] == [ + 'lvchange', + '--deltag', 'ceph.db_uuid=dbuuid', + '--deltag', 'ceph.db_device=db_dev', + '/dev/VolGroup/lv1'] + + assert self. mock_process_input[n-3] == [ + 'lvchange', + '--addtag', 'ceph.db_uuid=new-db-uuid', + '--addtag', 'ceph.db_device=/dev/VolGroup/lv2_new', + '/dev/VolGroup/lv1'] + + assert self. mock_process_input[n-2] == [ + 'lvchange', + '--addtag', 'ceph.osd_id=2', + '--addtag', 'ceph.type=db', + '--addtag', 'ceph.osd_fsid=1234', + '--addtag', 'ceph.cluster_name=ceph', + '--addtag', 'ceph.db_uuid=new-db-uuid', + '--addtag', 'ceph.db_device=/dev/VolGroup/lv2_new', + '/dev/VolGroup/lv2_new'] + + assert self. mock_process_input[n-1] == [ + 'ceph-bluestore-tool', + '--path', '/var/lib/ceph/osd/ceph-2', + '--dev-target', '/dev/VolGroup/lv2_new', + '--command', 'bluefs-bdev-migrate', + '--devs-source', '/var/lib/ceph/osd/ceph-2/block', + '--devs-source', '/var/lib/ceph/osd/ceph-2/block.db'] + + @patch('os.getuid') + def test_migrate_data_db_to_new_db_skip_wal(self, m_getuid, monkeypatch): + m_getuid.return_value = 0 + + source_tags = 'ceph.osd_id=2,ceph.type=data,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev' + source_db_tags = 'ceph.osd_id=2,ceph.type=db,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev' + source_wal_tags = 'ceph.osd_id=2,ceph.type=wal,ceph.osd_fsid=1234' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev' + + data_vol = api.Volume(lv_name='volume1', + lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', + lv_uuid='dbuuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2', + lv_tags=source_db_tags) + + wal_vol = api.Volume(lv_name='volume3', + lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv3', + lv_tags=source_wal_tags) + + self.mock_single_volumes = { + '/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol, + '/dev/VolGroup/lv3': wal_vol, + } + monkeypatch.setattr(migrate.api, 'get_first_lv', + self.mock_get_first_lv) + + self.mock_volume = api.Volume(lv_name='volume2_new', lv_uuid='new-db-uuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2_new', + lv_tags='') + monkeypatch.setattr(api, 'get_lv_by_fullname', + self.mock_get_lv_by_fullname) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + devices = [] + devices.append([Device('/dev/VolGroup/lv1'), 'block']) + devices.append([Device('/dev/VolGroup/lv2'), 'db']) + devices.append([Device('/dev/VolGroup/lv3'), 'wal']) + + monkeypatch.setattr(migrate, 'find_associated_devices', + lambda osd_id, osd_fsid: devices) + + monkeypatch.setattr("ceph_volume.systemd.systemctl.osd_is_active", + lambda id: False) + + monkeypatch.setattr(migrate, 'get_cluster_name', + lambda osd_id, osd_fsid: 'ceph') + monkeypatch.setattr(system, 'chown', lambda path: 0) + m = migrate.Migrate(argv=[ + '--osd-id', '2', + '--osd-fsid', '1234', + '--from', 'data', 'db', + '--target', 'vgname/new_wal']) + m.main() + + n = len(self.mock_process_input) + assert n >= 7 + + assert self. mock_process_input[n-7] == [ + 'lvchange', + '--deltag', 'ceph.osd_id=2', + '--deltag', 'ceph.type=db', + '--deltag', 'ceph.osd_fsid=1234', + '--deltag', 'ceph.cluster_name=ceph', + '--deltag', 'ceph.db_uuid=dbuuid', + '--deltag', 'ceph.db_device=db_dev', + '/dev/VolGroup/lv2'] + + assert self. mock_process_input[n-6] == [ + 'lvchange', + '--deltag', 'ceph.db_uuid=dbuuid', + '--deltag', 'ceph.db_device=db_dev', + '/dev/VolGroup/lv1'] + + assert self. mock_process_input[n-5] == [ + 'lvchange', + '--addtag', 'ceph.db_uuid=new-db-uuid', + '--addtag', 'ceph.db_device=/dev/VolGroup/lv2_new', + '/dev/VolGroup/lv1'] + + assert self. mock_process_input[n-4] == [ + 'lvchange', + '--deltag', 'ceph.db_uuid=dbuuid', + '--deltag', 'ceph.db_device=db_dev', + '/dev/VolGroup/lv3'] + + assert self. mock_process_input[n-3] == [ + 'lvchange', + '--addtag', 'ceph.db_uuid=new-db-uuid', + '--addtag', 'ceph.db_device=/dev/VolGroup/lv2_new', + '/dev/VolGroup/lv3'] + + assert self. mock_process_input[n-2] == [ + 'lvchange', + '--addtag', 'ceph.osd_id=2', + '--addtag', 'ceph.type=db', + '--addtag', 'ceph.osd_fsid=1234', + '--addtag', 'ceph.cluster_name=ceph', + '--addtag', 'ceph.db_uuid=new-db-uuid', + '--addtag', 'ceph.db_device=/dev/VolGroup/lv2_new', + '/dev/VolGroup/lv2_new'] + + assert self. mock_process_input[n-1] == [ + 'ceph-bluestore-tool', + '--path', '/var/lib/ceph/osd/ceph-2', + '--dev-target', '/dev/VolGroup/lv2_new', + '--command', 'bluefs-bdev-migrate', + '--devs-source', '/var/lib/ceph/osd/ceph-2/block', + '--devs-source', '/var/lib/ceph/osd/ceph-2/block.db'] + + @patch('os.getuid') + def test_migrate_data_db_wal_to_new_db(self, m_getuid, monkeypatch): + m_getuid.return_value = 0 + + source_tags = 'ceph.osd_id=2,ceph.type=data,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + source_db_tags = 'ceph.osd_id=2,ceph.type=db,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev' + source_wal_tags = 'ceph.osd_id=0,ceph.type=wal,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + + data_vol = api.Volume(lv_name='volume1', + lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', + lv_uuid='dbuuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2', + lv_tags=source_db_tags) + + wal_vol = api.Volume(lv_name='volume3', + lv_uuid='waluuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv3', + lv_tags=source_wal_tags) + + self.mock_single_volumes = { + '/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol, + '/dev/VolGroup/lv3': wal_vol, + } + monkeypatch.setattr(migrate.api, 'get_first_lv', + self.mock_get_first_lv) + + self.mock_volume = api.Volume(lv_name='volume2_new', lv_uuid='new-db-uuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2_new', + lv_tags='') + monkeypatch.setattr(api, 'get_lv_by_fullname', + self.mock_get_lv_by_fullname) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + devices = [] + devices.append([Device('/dev/VolGroup/lv1'), 'block']) + devices.append([Device('/dev/VolGroup/lv2'), 'db']) + devices.append([Device('/dev/VolGroup/lv3'), 'wal']) + + monkeypatch.setattr(migrate, 'find_associated_devices', + lambda osd_id, osd_fsid: devices) + + monkeypatch.setattr("ceph_volume.systemd.systemctl.osd_is_active", + lambda id: False) + + monkeypatch.setattr(migrate, 'get_cluster_name', + lambda osd_id, osd_fsid: 'ceph') + monkeypatch.setattr(system, 'chown', lambda path: 0) + m = migrate.Migrate(argv=[ + '--osd-id', '2', + '--osd-fsid', '1234', + '--from', 'data', 'db', 'wal', + '--target', 'vgname/new_wal']) + m.main() + + n = len(self.mock_process_input) + assert n >= 6 + + assert self. mock_process_input[n-6] == [ + 'lvchange', + '--deltag', 'ceph.osd_id=2', + '--deltag', 'ceph.type=db', + '--deltag', 'ceph.osd_fsid=1234', + '--deltag', 'ceph.cluster_name=ceph', + '--deltag', 'ceph.db_uuid=dbuuid', + '--deltag', 'ceph.db_device=db_dev', + '/dev/VolGroup/lv2'] + + assert self. mock_process_input[n-5] == [ + 'lvchange', + '--deltag', 'ceph.osd_id=0', + '--deltag', 'ceph.type=wal', + '--deltag', 'ceph.osd_fsid=1234', + '--deltag', 'ceph.cluster_name=ceph', + '--deltag', 'ceph.db_uuid=dbuuid', + '--deltag', 'ceph.db_device=db_dev', + '--deltag', 'ceph.wal_uuid=waluuid', + '--deltag', 'ceph.wal_device=wal_dev', + '/dev/VolGroup/lv3'] + + assert self. mock_process_input[n-4] == [ + 'lvchange', + '--deltag', 'ceph.db_uuid=dbuuid', + '--deltag', 'ceph.db_device=db_dev', + '--deltag', 'ceph.wal_uuid=waluuid', + '--deltag', 'ceph.wal_device=wal_dev', + '/dev/VolGroup/lv1'] + + assert self. mock_process_input[n-3] == [ + 'lvchange', + '--addtag', 'ceph.db_uuid=new-db-uuid', + '--addtag', 'ceph.db_device=/dev/VolGroup/lv2_new', + '/dev/VolGroup/lv1'] + + assert self. mock_process_input[n-2] == [ + 'lvchange', + '--addtag', 'ceph.osd_id=2', + '--addtag', 'ceph.type=db', + '--addtag', 'ceph.osd_fsid=1234', + '--addtag', 'ceph.cluster_name=ceph', + '--addtag', 'ceph.db_uuid=new-db-uuid', + '--addtag', 'ceph.db_device=/dev/VolGroup/lv2_new', + '/dev/VolGroup/lv2_new'] + + assert self. mock_process_input[n-1] == [ + 'ceph-bluestore-tool', + '--path', '/var/lib/ceph/osd/ceph-2', + '--dev-target', '/dev/VolGroup/lv2_new', + '--command', 'bluefs-bdev-migrate', + '--devs-source', '/var/lib/ceph/osd/ceph-2/block', + '--devs-source', '/var/lib/ceph/osd/ceph-2/block.db', + '--devs-source', '/var/lib/ceph/osd/ceph-2/block.wal'] + + @patch('os.getuid') + def test_dont_migrate_data_db_wal_to_new_data(self, + m_getuid, + monkeypatch, + capsys): + m_getuid.return_value = 0 + + source_tags = 'ceph.osd_id=2,ceph.type=data,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev' + source_db_tags = 'ceph.osd_id=2,ceph.type=db,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev' + + data_vol = api.Volume(lv_name='volume1', + lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', + lv_uuid='dbuuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2', + lv_tags=source_db_tags) + + self.mock_single_volumes = { + '/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol, + } + monkeypatch.setattr(migrate.api, 'get_first_lv', + self.mock_get_first_lv) + + self.mock_volume = api.Volume(lv_name='volume2_new', lv_uuid='new-db-uuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2_new', + lv_tags='') + monkeypatch.setattr(api, 'get_lv_by_fullname', + self.mock_get_lv_by_fullname) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + devices = [] + devices.append([Device('/dev/VolGroup/lv1'), 'block']) + devices.append([Device('/dev/VolGroup/lv2'), 'db']) + + monkeypatch.setattr(migrate, 'find_associated_devices', + lambda osd_id, osd_fsid: devices) + + monkeypatch.setattr("ceph_volume.systemd.systemctl.osd_is_active", + lambda id: False) + + monkeypatch.setattr(migrate, 'get_cluster_name', + lambda osd_id, osd_fsid: 'ceph') + monkeypatch.setattr(system, 'chown', lambda path: 0) + m = migrate.Migrate(argv=[ + '--osd-id', '2', + '--osd-fsid', '1234', + '--from', 'data', + '--target', 'vgname/new_data']) + + with pytest.raises(SystemExit) as error: + m.main() + stdout, stderr = capsys.readouterr() + expected = 'Unable to migrate to : vgname/new_data' + assert expected in str(error.value) + expected = 'Unable to determine new volume type,' + ' please use new-db or new-wal command before.' + assert expected in stderr + + @patch('os.getuid') + def test_dont_migrate_db_to_wal(self, + m_getuid, + monkeypatch, + capsys): + m_getuid.return_value = 0 + + source_tags = 'ceph.osd_id=2,ceph.type=data,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + source_db_tags = 'ceph.osd_id=2,ceph.type=db,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev' + source_wal_tags = 'ceph.osd_id=2,ceph.type=wal,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + + data_vol = api.Volume(lv_name='volume1', + lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', + lv_uuid='dbuuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2', + lv_tags=source_db_tags) + + wal_vol = api.Volume(lv_name='volume3', + lv_uuid='waluuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv3', + lv_tags=source_wal_tags) + + self.mock_single_volumes = { + '/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol, + '/dev/VolGroup/lv3': wal_vol, + } + monkeypatch.setattr(migrate.api, 'get_first_lv', + self.mock_get_first_lv) + + self.mock_volume = wal_vol + monkeypatch.setattr(api, 'get_lv_by_fullname', + self.mock_get_lv_by_fullname) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + devices = [] + devices.append([Device('/dev/VolGroup/lv1'), 'block']) + devices.append([Device('/dev/VolGroup/lv2'), 'db']) + devices.append([Device('/dev/VolGroup/lv3'), 'wal']) + + monkeypatch.setattr(migrate, 'find_associated_devices', + lambda osd_id, osd_fsid: devices) + + monkeypatch.setattr("ceph_volume.systemd.systemctl.osd_is_active", + lambda id: False) + + monkeypatch.setattr(migrate, 'get_cluster_name', + lambda osd_id, osd_fsid: 'ceph') + monkeypatch.setattr(system, 'chown', lambda path: 0) + m = migrate.Migrate(argv=[ + '--osd-id', '2', + '--osd-fsid', '1234', + '--from', 'db', + '--target', 'vgname/wal']) + + with pytest.raises(SystemExit) as error: + m.main() + stdout, stderr = capsys.readouterr() + expected = 'Unable to migrate to : vgname/wal' + assert expected in str(error.value) + expected = 'Migrate to WAL is not supported' + assert expected in stderr + + @patch('os.getuid') + def test_migrate_data_db_to_db(self, + m_getuid, + monkeypatch, + capsys): + m_getuid.return_value = 0 + + source_tags = 'ceph.osd_id=2,ceph.type=data,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + source_db_tags = 'ceph.osd_id=2,ceph.type=db,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev' + source_wal_tags = 'ceph.osd_id=2,ceph.type=wal,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + + data_vol = api.Volume(lv_name='volume1', + lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', + lv_uuid='dbuuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2', + lv_tags=source_db_tags) + + wal_vol = api.Volume(lv_name='volume3', + lv_uuid='waluuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv3', + lv_tags=source_wal_tags) + + self.mock_single_volumes = { + '/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol, + '/dev/VolGroup/lv3': wal_vol, + } + monkeypatch.setattr(migrate.api, 'get_first_lv', + self.mock_get_first_lv) + + self.mock_volume = db_vol + monkeypatch.setattr(api, 'get_lv_by_fullname', + self.mock_get_lv_by_fullname) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + devices = [] + devices.append([Device('/dev/VolGroup/lv1'), 'block']) + devices.append([Device('/dev/VolGroup/lv2'), 'db']) + devices.append([Device('/dev/VolGroup/lv3'), 'wal']) + + monkeypatch.setattr(migrate, 'find_associated_devices', + lambda osd_id, osd_fsid: devices) + + monkeypatch.setattr("ceph_volume.systemd.systemctl.osd_is_active", + lambda id: False) + + monkeypatch.setattr(migrate, 'get_cluster_name', + lambda osd_id, osd_fsid: 'ceph') + monkeypatch.setattr(system, 'chown', lambda path: 0) + m = migrate.Migrate(argv=[ + '--osd-id', '2', + '--osd-fsid', '1234', + '--from', 'db', 'data', + '--target', 'vgname/db']) + + m.main() + + n = len(self.mock_process_input) + assert n >= 1 + for s in self.mock_process_input: + print(s) + + assert self. mock_process_input[n-1] == [ + 'ceph-bluestore-tool', + '--path', '/var/lib/ceph/osd/ceph-2', + '--dev-target', '/var/lib/ceph/osd/ceph-2/block.db', + '--command', 'bluefs-bdev-migrate', + '--devs-source', '/var/lib/ceph/osd/ceph-2/block'] + + def test_migrate_data_db_to_db_active_systemd(self, is_root, monkeypatch, capsys): + source_tags = 'ceph.osd_id=2,ceph.type=data,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + source_db_tags = 'ceph.osd_id=2,ceph.type=db,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev' + source_wal_tags = 'ceph.osd_id=2,ceph.type=wal,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + + data_vol = api.Volume(lv_name='volume1', + lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', + lv_uuid='dbuuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2', + lv_tags=source_db_tags) + + wal_vol = api.Volume(lv_name='volume3', + lv_uuid='waluuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv3', + lv_tags=source_wal_tags) + + self.mock_single_volumes = { + '/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol, + '/dev/VolGroup/lv3': wal_vol, + } + monkeypatch.setattr(migrate.api, 'get_first_lv', + self.mock_get_first_lv) + + self.mock_volume = db_vol + monkeypatch.setattr(api, 'get_lv_by_fullname', + self.mock_get_lv_by_fullname) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + devices = [] + devices.append([Device('/dev/VolGroup/lv1'), 'block']) + devices.append([Device('/dev/VolGroup/lv2'), 'db']) + devices.append([Device('/dev/VolGroup/lv3'), 'wal']) + + monkeypatch.setattr(migrate, 'find_associated_devices', + lambda osd_id, osd_fsid: devices) + + monkeypatch.setattr("ceph_volume.systemd.systemctl.osd_is_active", + lambda id: True) + + monkeypatch.setattr(migrate, 'get_cluster_name', + lambda osd_id, osd_fsid: 'ceph') + monkeypatch.setattr(system, 'chown', lambda path: 0) + m = migrate.Migrate(argv=[ + '--osd-id', '2', + '--osd-fsid', '1234', + '--from', 'db', 'data', + '--target', 'vgname/db']) + + with pytest.raises(SystemExit) as error: + m.main() + + stdout, stderr = capsys.readouterr() + + assert 'Unable to migrate devices associated with OSD ID: 2' == str(error.value) + assert '--> OSD is running, stop it with: systemctl stop ceph-osd@2' == stderr.rstrip() + assert not stdout + + def test_migrate_data_db_to_db_no_systemd(self, is_root, monkeypatch): + source_tags = 'ceph.osd_id=2,ceph.type=data,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + source_db_tags = 'ceph.osd_id=2,ceph.type=db,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev' + source_wal_tags = 'ceph.osd_id=2,ceph.type=wal,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + + data_vol = api.Volume(lv_name='volume1', + lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', + lv_uuid='dbuuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2', + lv_tags=source_db_tags) + + wal_vol = api.Volume(lv_name='volume3', + lv_uuid='waluuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv3', + lv_tags=source_wal_tags) + + self.mock_single_volumes = { + '/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol, + '/dev/VolGroup/lv3': wal_vol, + } + monkeypatch.setattr(migrate.api, 'get_first_lv', + self.mock_get_first_lv) + + self.mock_volume = db_vol + monkeypatch.setattr(api, 'get_lv_by_fullname', + self.mock_get_lv_by_fullname) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + devices = [] + devices.append([Device('/dev/VolGroup/lv1'), 'block']) + devices.append([Device('/dev/VolGroup/lv2'), 'db']) + devices.append([Device('/dev/VolGroup/lv3'), 'wal']) + + monkeypatch.setattr(migrate, 'find_associated_devices', + lambda osd_id, osd_fsid: devices) + + monkeypatch.setattr(migrate, 'get_cluster_name', + lambda osd_id, osd_fsid: 'ceph') + monkeypatch.setattr(system, 'chown', lambda path: 0) + m = migrate.Migrate(argv=[ + '--osd-id', '2', + '--osd-fsid', '1234', + '--from', 'db', 'data', + '--target', 'vgname/db', + '--no-systemd']) + + m.main() + + n = len(self.mock_process_input) + assert n >= 1 + for s in self.mock_process_input: + print(s) + + assert self. mock_process_input[n-1] == [ + 'ceph-bluestore-tool', + '--path', '/var/lib/ceph/osd/ceph-2', + '--dev-target', '/var/lib/ceph/osd/ceph-2/block.db', + '--command', 'bluefs-bdev-migrate', + '--devs-source', '/var/lib/ceph/osd/ceph-2/block'] + + @patch('os.getuid') + def test_migrate_data_wal_to_db(self, + m_getuid, + monkeypatch, + capsys): + m_getuid.return_value = 0 + + source_tags = 'ceph.osd_id=2,ceph.type=data,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + source_db_tags = 'ceph.osd_id=2,ceph.type=db,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + source_wal_tags = 'ceph.osd_id=2,ceph.type=wal,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + + data_vol = api.Volume(lv_name='volume1', + lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', + lv_uuid='dbuuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2', + lv_tags=source_db_tags) + + wal_vol = api.Volume(lv_name='volume3', + lv_uuid='waluuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv3', + lv_tags=source_wal_tags) + + self.mock_single_volumes = { + '/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol, + '/dev/VolGroup/lv3': wal_vol, + } + monkeypatch.setattr(migrate.api, 'get_first_lv', + self.mock_get_first_lv) + + self.mock_volume = db_vol + monkeypatch.setattr(api, 'get_lv_by_fullname', + self.mock_get_lv_by_fullname) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + devices = [] + devices.append([Device('/dev/VolGroup/lv1'), 'block']) + devices.append([Device('/dev/VolGroup/lv2'), 'db']) + devices.append([Device('/dev/VolGroup/lv3'), 'wal']) + + monkeypatch.setattr(migrate, 'find_associated_devices', + lambda osd_id, osd_fsid: devices) + + monkeypatch.setattr("ceph_volume.systemd.systemctl.osd_is_active", + lambda id: False) + + monkeypatch.setattr(migrate, 'get_cluster_name', + lambda osd_id, osd_fsid: 'ceph') + monkeypatch.setattr(system, 'chown', lambda path: 0) + m = migrate.Migrate(argv=[ + '--osd-id', '2', + '--osd-fsid', '1234', + '--from', 'db', 'data', 'wal', + '--target', 'vgname/db']) + + m.main() + + n = len(self.mock_process_input) + assert n >= 1 + for s in self.mock_process_input: + print(s) + + assert self. mock_process_input[n-4] == [ + 'lvchange', + '--deltag', 'ceph.osd_id=2', + '--deltag', 'ceph.type=wal', + '--deltag', 'ceph.osd_fsid=1234', + '--deltag', 'ceph.cluster_name=ceph', + '--deltag', 'ceph.db_uuid=dbuuid', + '--deltag', 'ceph.db_device=db_dev', + '--deltag', 'ceph.wal_uuid=waluuid', + '--deltag', 'ceph.wal_device=wal_dev', + '/dev/VolGroup/lv3'] + assert self. mock_process_input[n-3] == [ + 'lvchange', + '--deltag', 'ceph.wal_uuid=waluuid', + '--deltag', 'ceph.wal_device=wal_dev', + '/dev/VolGroup/lv1'] + assert self. mock_process_input[n-2] == [ + 'lvchange', + '--deltag', 'ceph.wal_uuid=waluuid', + '--deltag', 'ceph.wal_device=wal_dev', + '/dev/VolGroup/lv2'] + assert self. mock_process_input[n-1] == [ + 'ceph-bluestore-tool', + '--path', '/var/lib/ceph/osd/ceph-2', + '--dev-target', '/var/lib/ceph/osd/ceph-2/block.db', + '--command', 'bluefs-bdev-migrate', + '--devs-source', '/var/lib/ceph/osd/ceph-2/block', + '--devs-source', '/var/lib/ceph/osd/ceph-2/block.wal'] + + def test_migrate_data_wal_to_db_active_systemd(self, is_root, monkeypatch, capsys): + source_tags = 'ceph.osd_id=2,ceph.type=data,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + source_db_tags = 'ceph.osd_id=2,ceph.type=db,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + source_wal_tags = 'ceph.osd_id=2,ceph.type=wal,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + + data_vol = api.Volume(lv_name='volume1', + lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', + lv_uuid='dbuuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2', + lv_tags=source_db_tags) + + wal_vol = api.Volume(lv_name='volume3', + lv_uuid='waluuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv3', + lv_tags=source_wal_tags) + + self.mock_single_volumes = { + '/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol, + '/dev/VolGroup/lv3': wal_vol, + } + monkeypatch.setattr(migrate.api, 'get_first_lv', + self.mock_get_first_lv) + + self.mock_volume = db_vol + monkeypatch.setattr(api, 'get_lv_by_fullname', + self.mock_get_lv_by_fullname) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + devices = [] + devices.append([Device('/dev/VolGroup/lv1'), 'block']) + devices.append([Device('/dev/VolGroup/lv2'), 'db']) + devices.append([Device('/dev/VolGroup/lv3'), 'wal']) + + monkeypatch.setattr(migrate, 'find_associated_devices', + lambda osd_id, osd_fsid: devices) + + monkeypatch.setattr("ceph_volume.systemd.systemctl.osd_is_active", + lambda id: True) + + monkeypatch.setattr(migrate, 'get_cluster_name', + lambda osd_id, osd_fsid: 'ceph') + monkeypatch.setattr(system, 'chown', lambda path: 0) + m = migrate.Migrate(argv=[ + '--osd-id', '2', + '--osd-fsid', '1234', + '--from', 'db', 'data', 'wal', + '--target', 'vgname/db']) + + with pytest.raises(SystemExit) as error: + m.main() + + stdout, stderr = capsys.readouterr() + + assert 'Unable to migrate devices associated with OSD ID: 2' == str(error.value) + assert '--> OSD is running, stop it with: systemctl stop ceph-osd@2' == stderr.rstrip() + assert not stdout + + def test_migrate_data_wal_to_db_no_systemd(self, is_root, monkeypatch): + source_tags = 'ceph.osd_id=2,ceph.type=data,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + source_db_tags = 'ceph.osd_id=2,ceph.type=db,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + source_wal_tags = 'ceph.osd_id=2,ceph.type=wal,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,ceph.db_uuid=dbuuid,ceph.db_device=db_dev,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + + data_vol = api.Volume(lv_name='volume1', + lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags=source_tags) + db_vol = api.Volume(lv_name='volume2', + lv_uuid='dbuuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv2', + lv_tags=source_db_tags) + + wal_vol = api.Volume(lv_name='volume3', + lv_uuid='waluuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv3', + lv_tags=source_wal_tags) + + self.mock_single_volumes = { + '/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv2': db_vol, + '/dev/VolGroup/lv3': wal_vol, + } + monkeypatch.setattr(migrate.api, 'get_first_lv', + self.mock_get_first_lv) + + self.mock_volume = db_vol + monkeypatch.setattr(api, 'get_lv_by_fullname', + self.mock_get_lv_by_fullname) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + devices = [] + devices.append([Device('/dev/VolGroup/lv1'), 'block']) + devices.append([Device('/dev/VolGroup/lv2'), 'db']) + devices.append([Device('/dev/VolGroup/lv3'), 'wal']) + + monkeypatch.setattr(migrate, 'find_associated_devices', + lambda osd_id, osd_fsid: devices) + + monkeypatch.setattr(migrate, 'get_cluster_name', + lambda osd_id, osd_fsid: 'ceph') + monkeypatch.setattr(system, 'chown', lambda path: 0) + m = migrate.Migrate(argv=[ + '--osd-id', '2', + '--osd-fsid', '1234', + '--from', 'db', 'data', 'wal', + '--target', 'vgname/db', + '--no-systemd']) + + m.main() + + n = len(self.mock_process_input) + assert n >= 1 + for s in self.mock_process_input: + print(s) + + assert self. mock_process_input[n-4] == [ + 'lvchange', + '--deltag', 'ceph.osd_id=2', + '--deltag', 'ceph.type=wal', + '--deltag', 'ceph.osd_fsid=1234', + '--deltag', 'ceph.cluster_name=ceph', + '--deltag', 'ceph.db_uuid=dbuuid', + '--deltag', 'ceph.db_device=db_dev', + '--deltag', 'ceph.wal_uuid=waluuid', + '--deltag', 'ceph.wal_device=wal_dev', + '/dev/VolGroup/lv3'] + assert self. mock_process_input[n-3] == [ + 'lvchange', + '--deltag', 'ceph.wal_uuid=waluuid', + '--deltag', 'ceph.wal_device=wal_dev', + '/dev/VolGroup/lv1'] + assert self. mock_process_input[n-2] == [ + 'lvchange', + '--deltag', 'ceph.wal_uuid=waluuid', + '--deltag', 'ceph.wal_device=wal_dev', + '/dev/VolGroup/lv2'] + assert self. mock_process_input[n-1] == [ + 'ceph-bluestore-tool', + '--path', '/var/lib/ceph/osd/ceph-2', + '--dev-target', '/var/lib/ceph/osd/ceph-2/block.db', + '--command', 'bluefs-bdev-migrate', + '--devs-source', '/var/lib/ceph/osd/ceph-2/block', + '--devs-source', '/var/lib/ceph/osd/ceph-2/block.wal'] diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/raw/test_list.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/raw/test_list.py new file mode 100644 index 000000000..d5ccee5c9 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/raw/test_list.py @@ -0,0 +1,235 @@ +import pytest +from mock.mock import patch +from ceph_volume.devices import raw + +# Sample lsblk output is below that overviews the test scenario. (--json output for reader clarity) +# - sda and all its children are used for the OS +# - sdb is a bluestore OSD with phantom Atari partitions +# - sdc is an empty disk +# - sdd has 2 LVM device children +# > lsblk --paths --json +# { +# "blockdevices": [ +# {"name": "/dev/sda", "maj:min": "8:0", "rm": "0", "size": "128G", "ro": "0", "type": "disk", "mountpoint": null, +# "children": [ +# {"name": "/dev/sda1", "maj:min": "8:1", "rm": "0", "size": "487M", "ro": "0", "type": "part", "mountpoint": null}, +# {"name": "/dev/sda2", "maj:min": "8:2", "rm": "0", "size": "1.9G", "ro": "0", "type": "part", "mountpoint": null}, +# {"name": "/dev/sda3", "maj:min": "8:3", "rm": "0", "size": "125.6G", "ro": "0", "type": "part", "mountpoint": "/etc/hosts"} +# ] +# }, +# {"name": "/dev/sdb", "maj:min": "8:16", "rm": "0", "size": "1T", "ro": "0", "type": "disk", "mountpoint": null, +# "children": [ +# {"name": "/dev/sdb2", "maj:min": "8:18", "rm": "0", "size": "48G", "ro": "0", "type": "part", "mountpoint": null}, +# {"name": "/dev/sdb3", "maj:min": "8:19", "rm": "0", "size": "6M", "ro": "0", "type": "part", "mountpoint": null} +# ] +# }, +# {"name": "/dev/sdc", "maj:min": "8:32", "rm": "0", "size": "1T", "ro": "0", "type": "disk", "mountpoint": null}, +# {"name": "/dev/sdd", "maj:min": "8:48", "rm": "0", "size": "1T", "ro": "0", "type": "disk", "mountpoint": null, +# "children": [ +# {"name": "/dev/mapper/ceph--osd--block--1", "maj:min": "253:0", "rm": "0", "size": "512G", "ro": "0", "type": "lvm", "mountpoint": null}, +# {"name": "/dev/mapper/ceph--osd--block--2", "maj:min": "253:1", "rm": "0", "size": "512G", "ro": "0", "type": "lvm", "mountpoint": null} +# ] +# } +# ] +# } + +def _devices_side_effect(): + return { + "/dev/sda": {}, + "/dev/sda1": {}, + "/dev/sda2": {}, + "/dev/sda3": {}, + "/dev/sdb": {}, + "/dev/sdb2": {}, + "/dev/sdb3": {}, + "/dev/sdc": {}, + "/dev/sdd": {}, + "/dev/mapper/ceph--osd--block--1": {}, + "/dev/mapper/ceph--osd--block--2": {}, + } + +def _lsblk_list_output(): + return [ + '/dev/sda', + '/dev/sda1', + '/dev/sda2', + '/dev/sda3', + '/dev/sdb', + '/dev/sdb2', + '/dev/sdb3', + '/dev/sdc', + '/dev/sdd', + '/dev/mapper/ceph--osd--block--1', + '/dev/mapper/ceph--osd--block--2', + ] + +# dummy lsblk output for device with optional parent output +def _lsblk_output(dev, parent=None): + if parent is None: + parent = "" + ret = 'NAME="{}" KNAME="{}" PKNAME="{}"'.format(dev, dev, parent) + return [ret] # needs to be in a list form + +def _bluestore_tool_label_output_sdb(): + return '''{ + "/dev/sdb": { + "osd_uuid": "sdb-uuid", + "size": 1099511627776, + "btime": "2021-07-23T16:02:22.809186+0000", + "description": "main", + "bfm_blocks": "268435456", + "bfm_blocks_per_key": "128", + "bfm_bytes_per_block": "4096", + "bfm_size": "1099511627776", + "bluefs": "1", + "ceph_fsid": "sdb-fsid", + "kv_backend": "rocksdb", + "magic": "ceph osd volume v026", + "mkfs_done": "yes", + "osd_key": "AQAO6PpgK+y4CBAAixq/X7OVimbaezvwD/cDmg==", + "ready": "ready", + "require_osd_release": "16", + "whoami": "0" + } +}''' + +def _bluestore_tool_label_output_sdb2(): + return '''{ + "/dev/sdb2": { + "osd_uuid": "sdb2-uuid", + "size": 1099511627776, + "btime": "2021-07-23T16:02:22.809186+0000", + "description": "main", + "bfm_blocks": "268435456", + "bfm_blocks_per_key": "128", + "bfm_bytes_per_block": "4096", + "bfm_size": "1099511627776", + "bluefs": "1", + "ceph_fsid": "sdb2-fsid", + "kv_backend": "rocksdb", + "magic": "ceph osd volume v026", + "mkfs_done": "yes", + "osd_key": "AQAO6PpgK+y4CBAAixq/X7OVimbaezvwD/cDmg==", + "ready": "ready", + "require_osd_release": "16", + "whoami": "2" + } +}''' + +def _bluestore_tool_label_output_dm_okay(): + return '''{ + "/dev/mapper/ceph--osd--block--1": { + "osd_uuid": "lvm-1-uuid", + "size": 549751619584, + "btime": "2021-07-23T16:04:37.881060+0000", + "description": "main", + "bfm_blocks": "134216704", + "bfm_blocks_per_key": "128", + "bfm_bytes_per_block": "4096", + "bfm_size": "549751619584", + "bluefs": "1", + "ceph_fsid": "lvm-1-fsid", + "kv_backend": "rocksdb", + "magic": "ceph osd volume v026", + "mkfs_done": "yes", + "osd_key": "AQCU6Ppgz+UcIRAAh6IUjtPjiXBlEXfwO8ixzw==", + "ready": "ready", + "require_osd_release": "16", + "whoami": "2" + } +}''' + +def _process_call_side_effect(command, **kw): + if "lsblk" in command: + if "/dev/" in command[-1]: + dev = command[-1] + if dev == "/dev/sda1" or dev == "/dev/sda2" or dev == "/dev/sda3": + return _lsblk_output(dev, parent="/dev/sda"), '', 0 + if dev == "/dev/sdb2" or dev == "/dev/sdb3": + return _lsblk_output(dev, parent="/dev/sdb"), '', 0 + if dev == "/dev/sda" or dev == "/dev/sdb" or dev == "/dev/sdc" or dev == "/dev/sdd": + return _lsblk_output(dev), '', 0 + if "mapper" in dev: + return _lsblk_output(dev, parent="/dev/sdd"), '', 0 + pytest.fail('dev {} needs behavior specified for it'.format(dev)) + if "/dev/" not in command: + return _lsblk_list_output(), '', 0 + pytest.fail('command {} needs behavior specified for it'.format(command)) + + if "ceph-bluestore-tool" in command: + if "/dev/sdb" in command: + # sdb is a bluestore OSD + return _bluestore_tool_label_output_sdb(), '', 0 + if "/dev/sdb2" in command: + # sdb2 is a phantom atari partition that appears to have some valid bluestore info + return _bluestore_tool_label_output_sdb2(), '', 0 + if "/dev/mapper/ceph--osd--block--1" in command: + # dm device 1 is a valid bluestore OSD (the other is corrupted/invalid) + return _bluestore_tool_label_output_dm_okay(), '', 0 + # sda and children, sdb's children, sdc, sdd, dm device 2 all do NOT have bluestore OSD data + return [], 'fake No such file or directory error', 1 + pytest.fail('command {} needs behavior specified for it'.format(command)) + +def _has_bluestore_label_side_effect(disk_path): + if "/dev/sda" in disk_path: + return False # disk and all children are for the OS + if disk_path == "/dev/sdb": + return True # sdb is a valid bluestore OSD + if disk_path == "/dev/sdb2": + return True # sdb2 appears to be a valid bluestore OSD even though it should not be + if disk_path == "/dev/sdc": + return False # empty disk + if disk_path == "/dev/sdd": + return False # has LVM subdevices + if disk_path == "/dev/mapper/ceph--osd--block--1": + return True # good OSD + if disk_path == "/dev/mapper/ceph--osd--block--2": + return False # corrupted + pytest.fail('device {} needs behavior specified for it'.format(disk_path)) + +class TestList(object): + + @patch('ceph_volume.util.device.disk.get_devices') + @patch('ceph_volume.util.disk.has_bluestore_label') + @patch('ceph_volume.process.call') + def test_raw_list(self, patched_call, patched_bluestore_label, patched_get_devices): + raw.list.logger.setLevel("DEBUG") + patched_call.side_effect = _process_call_side_effect + patched_bluestore_label.side_effect = _has_bluestore_label_side_effect + patched_get_devices.side_effect = _devices_side_effect + + result = raw.list.List([]).generate() + patched_call.assert_any_call(['lsblk', '--paths', '--output=NAME', '--noheadings', '--list']) + assert len(result) == 2 + + sdb = result['sdb-uuid'] + assert sdb['osd_uuid'] == 'sdb-uuid' + assert sdb['osd_id'] == 0 + assert sdb['device'] == '/dev/sdb' + assert sdb['ceph_fsid'] == 'sdb-fsid' + assert sdb['type'] == 'bluestore' + + lvm1 = result['lvm-1-uuid'] + assert lvm1['osd_uuid'] == 'lvm-1-uuid' + assert lvm1['osd_id'] == 2 + assert lvm1['device'] == '/dev/mapper/ceph--osd--block--1' + assert lvm1['ceph_fsid'] == 'lvm-1-fsid' + assert lvm1['type'] == 'bluestore' + + @patch('ceph_volume.util.device.disk.get_devices') + @patch('ceph_volume.util.disk.has_bluestore_label') + @patch('ceph_volume.process.call') + def test_raw_list_with_OSError(self, patched_call, patched_bluestore_label, patched_get_devices): + def _has_bluestore_label_side_effect_with_OSError(device_path): + if device_path == "/dev/sdd": + raise OSError('fake OSError') + return _has_bluestore_label_side_effect(device_path) + + raw.list.logger.setLevel("DEBUG") + patched_call.side_effect = _process_call_side_effect + patched_bluestore_label.side_effect = _has_bluestore_label_side_effect_with_OSError + patched_get_devices.side_effect = _devices_side_effect + + result = raw.list.List([]).generate() + assert len(result) == 1 + assert 'sdb-uuid' in result diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini b/ceph/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini index f7969fe9b..508d1b4c6 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini @@ -12,11 +12,9 @@ whitelist_externals = sleep passenv=* setenv= - ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config - ANSIBLE_ACTION_PLUGINS = {envdir}/tmp/ceph-ansible/plugins/actions + ANSIBLE_CONFIG = {envdir}/tmp/ceph-ansible/ansible.cfg + ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config -o ControlMaster=auto -o ControlPersist=600s -o PreferredAuthentications=publickey ANSIBLE_STDOUT_CALLBACK = debug - ANSIBLE_RETRY_FILES_ENABLED = False - ANSIBLE_SSH_RETRIES = 5 VAGRANT_CWD = {changedir} CEPH_VOLUME_DEBUG = 1 DEBIAN_FRONTEND=noninteractive @@ -53,7 +51,7 @@ commands= ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/setup.yml # test cluster state using testinfra - py.test -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests + py.test --reruns 5 --reruns-delay 10 -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests # reboot all vms - attempt bash {toxinidir}/../scripts/vagrant_reload.sh {env:VAGRANT_UP_FLAGS:"--no-provision"} {posargs:--provider=virtualbox} @@ -62,13 +60,13 @@ commands= sleep 30 # retest to ensure cluster came back up correctly after rebooting - py.test -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests + py.test --reruns 5 --reruns-delay 10 -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests # destroy an OSD, zap it's device and recreate it using it's ID ansible-playbook -vv -i {changedir}/hosts {changedir}/test.yml # retest to ensure cluster came back up correctly - py.test -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests + py.test --reruns 5 --reruns-delay 10 -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests # test zap OSDs by ID ansible-playbook -vv -i {changedir}/hosts {changedir}/test_zap.yml diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml index a05eef6ed..0a47b5eb8 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml @@ -15,12 +15,25 @@ - hosts: mons become: yes tasks: + - name: mark osds down + command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}" + with_items: + - 0 + - 2 - name: destroy osd.2 command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - name: destroy osd.0 command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - hosts: osds become: yes @@ -68,9 +81,15 @@ - hosts: mons become: yes tasks: + - name: mark osds down + command: "ceph --cluster {{ cluster }} osd down osd.0" - name: destroy osd.0 command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - hosts: osds diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml index f0408736e..21eff00fa 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/filestore/dmcrypt/test.yml @@ -17,13 +17,25 @@ - hosts: mons become: yes tasks: + - name: mark osds down + command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}" + with_items: + - 0 + - 2 - name: destroy osd.2 command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - name: destroy osd.0 command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it" - + register: result + retries: 30 + delay: 1 + until: result is succeeded - hosts: osds become: yes diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml index 2cf83e477..97d77a7f4 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml @@ -17,12 +17,25 @@ - hosts: mons become: yes tasks: + - name: mark osds down + command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}" + with_items: + - 0 + - 2 - name: destroy osd.2 command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - name: destroy osd.0 command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - hosts: osds diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml index 42ee40a1b..aca1f40a6 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml @@ -17,13 +17,25 @@ - hosts: mons become: yes tasks: + - name: mark osds down + command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}" + with_items: + - 0 + - 2 - name: destroy osd.2 command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it" + register: result + retries: 30 + delay: 1 + until: result is succeeded - name: destroy osd.0 command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it" - + register: result + retries: 30 + delay: 1 + until: result is succeeded - hosts: osds become: yes diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini index 2b63875bf..bec30e6d7 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini @@ -12,11 +12,9 @@ whitelist_externals = sleep passenv=* setenv= - ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config - ANSIBLE_ACTION_PLUGINS = {envdir}/tmp/ceph-ansible/plugins/actions + ANSIBLE_CONFIG = {envdir}/tmp/ceph-ansible/ansible.cfg + ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config -o ControlMaster=auto -o ControlPersist=600s -o PreferredAuthentications=publickey ANSIBLE_STDOUT_CALLBACK = debug - ANSIBLE_RETRY_FILES_ENABLED = False - ANSIBLE_SSH_RETRIES = 5 VAGRANT_CWD = {changedir} CEPH_VOLUME_DEBUG = 1 DEBIAN_FRONTEND=noninteractive @@ -53,7 +51,7 @@ commands= ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/setup.yml # test cluster state using testinfra - py.test -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests + py.test --reruns 5 --reruns-delay 10 -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests # reboot all vms - attempt bash {toxinidir}/../scripts/vagrant_reload.sh {env:VAGRANT_UP_FLAGS:"--no-provision"} {posargs:--provider=virtualbox} @@ -62,12 +60,12 @@ commands= sleep 30 # retest to ensure cluster came back up correctly after rebooting - py.test -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests + py.test --reruns 5 --reruns-delay 10 -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests # destroy an OSD, zap it's device and recreate it using it's ID ansible-playbook -vv -i {changedir}/hosts {changedir}/test.yml # retest to ensure cluster came back up correctly - py.test -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests + py.test --reruns 5 --reruns-delay 10 -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests vagrant destroy {env:VAGRANT_DESTROY_FLAGS:"--force"} diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/playbooks/deploy.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/playbooks/deploy.yml index 0c1d13f8f..e5185e3fc 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/playbooks/deploy.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/playbooks/deploy.yml @@ -75,8 +75,8 @@ - name: install required packages for fedora > 23 raw: sudo dnf -y install python2-dnf libselinux-python ntp when: - - ansible_distribution == 'Fedora' - - ansible_distribution_major_version|int >= 23 + - ansible_facts['distribution'] == 'Fedora' + - ansible_facts['distribution_major_version']|int >= 23 - name: check if it is atomic host stat: @@ -120,7 +120,7 @@ dest: "/usr/lib/python3.6/site-packages" use_ssh_args: true when: - - ansible_os_family == "RedHat" + - ansible_facts['os_family'] == "RedHat" - inventory_hostname in groups.get(osd_group_name, []) - name: rsync ceph-volume to test nodes on ubuntu @@ -129,7 +129,7 @@ dest: "/usr/lib/python2.7/dist-packages" use_ssh_args: true when: - - ansible_os_family == "Debian" + - ansible_facts['os_family'] == "Debian" - inventory_hostname in groups.get(osd_group_name, []) - name: run ceph-config role diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini index c3b7d3648..1fdfe26a8 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini @@ -12,11 +12,9 @@ whitelist_externals = cp passenv=* setenv= - ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config - ANSIBLE_ACTION_PLUGINS = {envdir}/tmp/ceph-ansible/plugins/actions + ANSIBLE_CONFIG = {envdir}/tmp/ceph-ansible/ansible.cfg + ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config -o ControlMaster=auto -o ControlPersist=600s -o PreferredAuthentications=publickey ANSIBLE_STDOUT_CALLBACK = debug - ANSIBLE_RETRY_FILES_ENABLED = False - ANSIBLE_SSH_RETRIES = 5 VAGRANT_CWD = {changedir} CEPH_VOLUME_DEBUG = 1 DEBIAN_FRONTEND=noninteractive @@ -43,7 +41,7 @@ commands= ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/setup.yml # test cluster state testinfra - py.test -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests + py.test --reruns 5 --reruns-delay 10 -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests # make ceph-volume simple take over all the OSDs that got deployed, disabling ceph-disk ansible-playbook -vv -i {changedir}/hosts {changedir}/test.yml @@ -55,6 +53,6 @@ commands= sleep 120 # retest to ensure cluster came back up correctly after rebooting - py.test -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests + py.test --reruns 5 --reruns-delay 10 -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests vagrant destroy {env:VAGRANT_DESTROY_FLAGS:"--force"} diff --git a/ceph/src/ceph-volume/ceph_volume/tests/util/test_arg_validators.py b/ceph/src/ceph-volume/ceph_volume/tests/util/test_arg_validators.py index d4565ef4d..1e8a49e26 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/util/test_arg_validators.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/util/test_arg_validators.py @@ -80,10 +80,10 @@ class TestValidDevice(object): def setup(self): self.validator = arg_validators.ValidDevice() - def test_path_is_valid(self, fake_call): + def test_path_is_valid(self, fake_call, patch_bluestore_label): result = self.validator('/') assert result.abspath == '/' - def test_path_is_invalid(self, fake_call): + def test_path_is_invalid(self, fake_call, patch_bluestore_label): with pytest.raises(argparse.ArgumentError): self.validator('/device/does/not/exist') diff --git a/ceph/src/ceph-volume/ceph_volume/tests/util/test_device.py b/ceph/src/ceph-volume/ceph_volume/tests/util/test_device.py index 0df5ac61e..a9d818d38 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/util/test_device.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/util/test_device.py @@ -2,6 +2,7 @@ import pytest from copy import deepcopy from ceph_volume.util import device from ceph_volume.api import lvm as api +from mock.mock import patch, mock_open class TestDevice(object): @@ -124,7 +125,7 @@ class TestDevice(object): def test_is_partition(self, device_info): data = {"/dev/sda1": {"foo": "bar"}} - lsblk = {"TYPE": "part"} + lsblk = {"TYPE": "part", "PKNAME": "sda"} device_info(devices=data, lsblk=lsblk) disk = device.Device("/dev/sda1") assert disk.is_partition @@ -138,14 +139,14 @@ class TestDevice(object): def test_is_not_lvm_memeber(self, device_info): data = {"/dev/sda1": {"foo": "bar"}} - lsblk = {"TYPE": "part"} + lsblk = {"TYPE": "part", "PKNAME": "sda"} device_info(devices=data, lsblk=lsblk) disk = device.Device("/dev/sda1") assert not disk.is_lvm_member def test_is_lvm_memeber(self, device_info): data = {"/dev/sda1": {"foo": "bar"}} - lsblk = {"TYPE": "part"} + lsblk = {"TYPE": "part", "PKNAME": "sda"} device_info(devices=data, lsblk=lsblk) disk = device.Device("/dev/sda1") assert not disk.is_lvm_member @@ -258,6 +259,14 @@ class TestDevice(object): assert not disk.available assert "Has BlueStore device label" in disk.rejected_reasons + def test_reject_device_with_oserror(self, monkeypatch, patch_bluestore_label, device_info): + patch_bluestore_label.side_effect = OSError('test failure') + lsblk = {"TYPE": "disk"} + device_info(lsblk=lsblk) + disk = device.Device("/dev/sda") + assert not disk.available + assert "Failed to determine if device is BlueStore" in disk.rejected_reasons + @pytest.mark.usefixtures("device_info_not_ceph_disk_member", "disable_kernel_queries") def test_is_not_ceph_disk_member_lsblk(self, patch_bluestore_label): @@ -306,7 +315,7 @@ class TestDevice(object): def test_used_by_ceph(self, device_info, monkeypatch, ceph_type): data = {"/dev/sda": {"foo": "bar"}} - lsblk = {"TYPE": "part"} + lsblk = {"TYPE": "part", "PKNAME": "sda"} FooPVolume = api.PVolume(pv_name='/dev/sda', pv_uuid="0000", lv_uuid="0000", pv_tags={}, vg_name="vg") pvolumes = [] @@ -333,7 +342,7 @@ class TestDevice(object): pvolumes = [] pvolumes.append(FooPVolume) data = {"/dev/sda": {"foo": "bar"}} - lsblk = {"TYPE": "part"} + lsblk = {"TYPE": "part", "PKNAME": "sda"} lv_data = {"lv_path": "vg/lv", "vg_name": "vg", "lv_uuid": "0000", "tags": {"ceph.osd_id": 0, "ceph.type": "journal"}} monkeypatch.setattr(api, 'get_pvs', lambda **kwargs: pvolumes) @@ -348,31 +357,41 @@ class TestDevice(object): disk = device.Device("/dev/sda") assert disk._get_device_id() == 'ID_VENDOR_ID_MODEL_ID_SCSI_SERIAL' + def test_has_bluestore_label(self): + # patch device.Device __init__ function to do nothing since we want to only test the + # low-level behavior of has_bluestore_label + with patch.object(device.Device, "__init__", lambda self, path, with_lsm=False: None): + disk = device.Device("/dev/sda") + disk.abspath = "/dev/sda" + with patch('builtins.open', mock_open(read_data=b'bluestore block device\n')): + assert disk.has_bluestore_label + with patch('builtins.open', mock_open(read_data=b'not a bluestore block device\n')): + assert not disk.has_bluestore_label class TestDeviceEncryption(object): def test_partition_is_not_encrypted_lsblk(self, device_info): - lsblk = {'TYPE': 'part', 'FSTYPE': 'xfs'} + lsblk = {'TYPE': 'part', 'FSTYPE': 'xfs', 'PKNAME': 'sda'} device_info(lsblk=lsblk) disk = device.Device("/dev/sda") assert disk.is_encrypted is False def test_partition_is_encrypted_lsblk(self, device_info): - lsblk = {'TYPE': 'part', 'FSTYPE': 'crypto_LUKS'} + lsblk = {'TYPE': 'part', 'FSTYPE': 'crypto_LUKS', 'PKNAME': 'sda'} device_info(lsblk=lsblk) disk = device.Device("/dev/sda") assert disk.is_encrypted is True def test_partition_is_not_encrypted_blkid(self, device_info): - lsblk = {'TYPE': 'part'} + lsblk = {'TYPE': 'part', 'PKNAME': 'sda'} blkid = {'TYPE': 'ceph data'} device_info(lsblk=lsblk, blkid=blkid) disk = device.Device("/dev/sda") assert disk.is_encrypted is False def test_partition_is_encrypted_blkid(self, device_info): - lsblk = {'TYPE': 'part'} + lsblk = {'TYPE': 'part', 'PKNAME': 'sda'} blkid = {'TYPE': 'crypto_LUKS'} device_info(lsblk=lsblk, blkid=blkid) disk = device.Device("/dev/sda") diff --git a/ceph/src/ceph-volume/ceph_volume/util/device.py b/ceph/src/ceph-volume/ceph_volume/util/device.py index deed06436..9a455883e 100644 --- a/ceph/src/ceph-volume/ceph_volume/util/device.py +++ b/ceph/src/ceph-volume/ceph_volume/util/device.py @@ -1,13 +1,18 @@ # -*- coding: utf-8 -*- +import logging import os from functools import total_ordering -from ceph_volume import sys_info, process +from ceph_volume import sys_info from ceph_volume.api import lvm from ceph_volume.util import disk, system from ceph_volume.util.lsmdisk import LSMDisk from ceph_volume.util.constants import ceph_disk_guids + +logger = logging.getLogger(__name__) + + report_template = """ {dev:<25} {size:<12} {rot!s:<7} {available!s:<9} {model}""" @@ -319,6 +324,12 @@ class Device(object): def size(self): return self.sys_api['size'] + @property + def parent_device(self): + if 'PKNAME' in self.disk_api: + return '/dev/%s' % self.disk_api['PKNAME'] + return None + @property def lvm_size(self): """ @@ -348,12 +359,7 @@ class Device(object): @property def has_bluestore_label(self): - out, err, ret = process.call([ - 'ceph-bluestore-tool', 'show-label', - '--dev', self.abspath], verbose_on_failure=False) - if ret: - return False - return True + return disk.has_bluestore_label(self.abspath) @property def is_mapper(self): @@ -476,8 +482,26 @@ class Device(object): rejected.append("Device type is not acceptable. It should be raw device or partition") if self.is_ceph_disk_member: rejected.append("Used by ceph-disk") - if self.has_bluestore_label: - rejected.append('Has BlueStore device label') + + try: + if self.has_bluestore_label: + rejected.append('Has BlueStore device label') + except OSError as e: + # likely failed to open the device. assuming it is BlueStore is the safest option + # so that a possibly-already-existing OSD doesn't get overwritten + logger.error('failed to determine if device {} is BlueStore. device should not be used to avoid false negatives. err: {}'.format(self.abspath, e)) + rejected.append('Failed to determine if device is BlueStore') + + if self.is_partition: + try: + if disk.has_bluestore_label(self.parent_device): + rejected.append('Parent has BlueStore device label') + except OSError as e: + # likely failed to open the device. assuming the parent is BlueStore is the safest + # option so that a possibly-already-existing OSD doesn't get overwritten + logger.error('failed to determine if partition {} (parent: {}) has a BlueStore parent. partition should not be used to avoid false negatives. err: {}'.format(self.abspath, self.parent_device, e)) + rejected.append('Failed to determine if parent device is BlueStore') + if self.has_gpt_headers: rejected.append('Has GPT headers') return rejected diff --git a/ceph/src/ceph-volume/ceph_volume/util/disk.py b/ceph/src/ceph-volume/ceph_volume/util/disk.py index fa79080e4..3d9e19c3e 100644 --- a/ceph/src/ceph-volume/ceph_volume/util/disk.py +++ b/ceph/src/ceph-volume/ceph_volume/util/disk.py @@ -134,14 +134,13 @@ def remove_partition(device): :param device: A ``Device()`` object """ - parent_device = '/dev/%s' % device.disk_api['PKNAME'] udev_info = udevadm_property(device.abspath) partition_number = udev_info.get('ID_PART_ENTRY_NUMBER') if not partition_number: raise RuntimeError('Unable to detect the partition number for device: %s' % device.abspath) process.run( - ['parted', parent_device, '--script', '--', 'rm', partition_number] + ['parted', device.parent_device, '--script', '--', 'rm', partition_number] ) @@ -802,3 +801,17 @@ def get_devices(_sys_block_path='/sys/block'): device_facts[diskname] = metadata return device_facts + +def has_bluestore_label(device_path): + isBluestore = False + bluestoreDiskSignature = 'bluestore block device' # 22 bytes long + + # throws OSError on failure + logger.info("opening device {} to check for BlueStore label".format(device_path)) + with open(device_path, "rb") as fd: + # read first 22 bytes looking for bluestore disk signature + signature = fd.read(22) + if signature.decode('ascii', 'replace') == bluestoreDiskSignature: + isBluestore = True + + return isBluestore diff --git a/ceph/src/ceph-volume/ceph_volume/util/system.py b/ceph/src/ceph-volume/ceph_volume/util/system.py index 499862337..d0d6545d3 100644 --- a/ceph/src/ceph-volume/ceph_volume/util/system.py +++ b/ceph/src/ceph-volume/ceph_volume/util/system.py @@ -260,6 +260,7 @@ def get_mounts(devices=False, paths=False, realpath=False): - tmpfs - devtmpfs + - /dev/root If ``devices`` is set to ``True`` the mapping will be a device-to-path(s), if ``paths`` is set to ``True`` then the mapping will be @@ -270,7 +271,7 @@ def get_mounts(devices=False, paths=False, realpath=False): """ devices_mounted = {} paths_mounted = {} - do_not_skip = ['tmpfs', 'devtmpfs'] + do_not_skip = ['tmpfs', 'devtmpfs', '/dev/root'] default_to_devices = devices is False and paths is False with open(PROCDIR + '/mounts', 'rb') as mounts: diff --git a/ceph/src/cephadm/cephadm b/ceph/src/cephadm/cephadm index 92bb8d617..13f0f6e6a 100755 --- a/ceph/src/cephadm/cephadm +++ b/ceph/src/cephadm/cephadm @@ -31,7 +31,7 @@ from contextlib import redirect_stdout import ssl from enum import Enum -from typing import Dict, List, Tuple, Optional, Union, Any, NoReturn, Callable, IO +from typing import Dict, List, Tuple, Optional, Union, Any, NoReturn, Callable, IO, Sequence, TypeVar, cast, Set import re import uuid @@ -45,14 +45,18 @@ from urllib.error import HTTPError from urllib.request import urlopen from pathlib import Path +FuncT = TypeVar('FuncT', bound=Callable) + # Default container images ----------------------------------------------------- -DEFAULT_IMAGE = 'docker.io/ceph/ceph:v16' +DEFAULT_IMAGE = 'quay.io/ceph/ceph:v16' DEFAULT_IMAGE_IS_MASTER = False DEFAULT_IMAGE_RELEASE = 'pacific' -DEFAULT_PROMETHEUS_IMAGE = 'docker.io/prom/prometheus:v2.18.1' -DEFAULT_NODE_EXPORTER_IMAGE = 'docker.io/prom/node-exporter:v0.18.1' -DEFAULT_GRAFANA_IMAGE = 'docker.io/ceph/ceph-grafana:6.7.4' -DEFAULT_ALERT_MANAGER_IMAGE = 'docker.io/prom/alertmanager:v0.20.0' +DEFAULT_PROMETHEUS_IMAGE = 'quay.io/prometheus/prometheus:v2.18.1' +DEFAULT_NODE_EXPORTER_IMAGE = 'quay.io/prometheus/node-exporter:v0.18.1' +DEFAULT_ALERT_MANAGER_IMAGE = 'quay.io/prometheus/alertmanager:v0.20.0' +DEFAULT_GRAFANA_IMAGE = 'quay.io/ceph/ceph-grafana:6.7.4' +DEFAULT_HAPROXY_IMAGE = 'docker.io/library/haproxy:2.3' +DEFAULT_KEEPALIVED_IMAGE = 'docker.io/arcts/keepalived' DEFAULT_REGISTRY = 'docker.io' # normalize unqualified digests to this # ------------------------------------------------------------------------------ @@ -104,7 +108,7 @@ cached_stdin = None class BaseConfig: - def __init__(self): + def __init__(self) -> None: self.image: str = '' self.docker: bool = False self.data_dir: str = DATA_DIR @@ -122,7 +126,7 @@ class BaseConfig: self.container_init: bool = CONTAINER_INIT self.container_engine: Optional[ContainerEngine] = None - def set_from_args(self, args: argparse.Namespace): + def set_from_args(self, args: argparse.Namespace) -> None: argdict: Dict[str, Any] = vars(args) for k, v in argdict.items(): if hasattr(self, k): @@ -131,7 +135,7 @@ class BaseConfig: class CephadmContext: - def __init__(self): + def __init__(self) -> None: self.__dict__['_args'] = None self.__dict__['_conf'] = BaseConfig() @@ -163,28 +167,29 @@ class CephadmContext: class ContainerEngine: - def __init__(self): + def __init__(self) -> None: self.path = find_program(self.EXE) + @classmethod @property - def EXE(self) -> str: + def EXE(cls) -> str: raise NotImplementedError() class Podman(ContainerEngine): EXE = 'podman' - def __init__(self): + def __init__(self) -> None: super().__init__() - self._version = None + self._version: Optional[Tuple[int, ...]] = None @property - def version(self): + def version(self) -> Tuple[int, ...]: if self._version is None: raise RuntimeError('Please call `get_version` first') return self._version - def get_version(self, ctx: CephadmContext): + def get_version(self, ctx: CephadmContext) -> None: out, _, _ = call_throws(ctx, [self.path, 'version', '--format', '{{.Client.Version}}']) self._version = _parse_podman_version(out) @@ -639,7 +644,7 @@ class HAproxy(object): """Defines an HAproxy container""" daemon_type = 'haproxy' required_files = ['haproxy.cfg'] - default_image = 'haproxy' + default_image = DEFAULT_HAPROXY_IMAGE def __init__(self, ctx: CephadmContext, @@ -702,7 +707,7 @@ class HAproxy(object): cname = '%s-%s' % (cname, desc) return cname - def extract_uid_gid_haproxy(self): + def extract_uid_gid_haproxy(self) -> Tuple[int, int]: # better directory for this? return extract_uid_gid(self.ctx, file_path='/var/lib') @@ -726,7 +731,7 @@ class Keepalived(object): """Defines an Keepalived container""" daemon_type = 'keepalived' required_files = ['keepalived.conf'] - default_image = 'arcts/keepalived' + default_image = DEFAULT_KEEPALIVED_IMAGE def __init__(self, ctx: CephadmContext, @@ -805,7 +810,7 @@ class Keepalived(object): 'net.ipv4.ip_nonlocal_bind = 1', ] - def extract_uid_gid_keepalived(self): + def extract_uid_gid_keepalived(self) -> Tuple[int, int]: # better directory for this? return extract_uid_gid(self.ctx, file_path='/var/lib') @@ -1070,34 +1075,34 @@ class Timeout(TimeoutError): seconds. """ - def __init__(self, lock_file): + def __init__(self, lock_file: str) -> None: """ """ #: The path of the file lock. self.lock_file = lock_file return None - def __str__(self): + def __str__(self) -> str: temp = "The file lock '{}' could not be acquired."\ .format(self.lock_file) return temp class _Acquire_ReturnProxy(object): - def __init__(self, lock): + def __init__(self, lock: 'FileLock') -> None: self.lock = lock return None - def __enter__(self): + def __enter__(self) -> 'FileLock': return self.lock - def __exit__(self, exc_type, exc_value, traceback): + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: self.lock.release() return None class FileLock(object): - def __init__(self, ctx: CephadmContext, name, timeout=-1): + def __init__(self, ctx: CephadmContext, name: str, timeout: int = -1) -> None: if not os.path.exists(LOCK_DIR): os.mkdir(LOCK_DIR, 0o700) self._lock_file = os.path.join(LOCK_DIR, name + '.lock') @@ -1116,10 +1121,10 @@ class FileLock(object): return None @property - def is_locked(self): + def is_locked(self) -> bool: return self._lock_file_fd is not None - def acquire(self, timeout=None, poll_intervall=0.05): + def acquire(self, timeout: Optional[int] = None, poll_intervall: float = 0.05) -> _Acquire_ReturnProxy: """ Acquires the file lock or fails with a :exc:`Timeout` error. .. code-block:: python @@ -1186,7 +1191,7 @@ class FileLock(object): raise return _Acquire_ReturnProxy(lock=self) - def release(self, force=False): + def release(self, force: bool = False) -> None: """ Releases the file lock. Please note, that the lock is only completly released, if the lock @@ -1200,29 +1205,32 @@ class FileLock(object): self._lock_counter -= 1 if self._lock_counter == 0 or force: - lock_id = id(self) - lock_filename = self._lock_file + # lock_id = id(self) + # lock_filename = self._lock_file - logger.debug('Releasing lock %s on %s', lock_id, lock_filename) + # Can't log in shutdown: + # File "/usr/lib64/python3.9/logging/__init__.py", line 1175, in _open + # NameError: name 'open' is not defined + # logger.debug('Releasing lock %s on %s', lock_id, lock_filename) self._release() self._lock_counter = 0 - logger.debug('Lock %s released on %s', lock_id, lock_filename) + # logger.debug('Lock %s released on %s', lock_id, lock_filename) return None - def __enter__(self): + def __enter__(self) -> 'FileLock': self.acquire() return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: self.release() return None - def __del__(self): + def __del__(self) -> None: self.release(force=True) return None - def _acquire(self): + def _acquire(self) -> None: open_mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC fd = os.open(self._lock_file, open_mode) @@ -1234,7 +1242,7 @@ class FileLock(object): self._lock_file_fd = fd return None - def _release(self): + def _release(self) -> None: # Do not remove the lockfile: # # https://github.com/benediktschmitt/py-filelock/issues/31 @@ -1276,7 +1284,7 @@ if sys.version_info < (3, 8): on amount of spawn processes. """ - def __init__(self): + def __init__(self) -> None: self._pid_counter = itertools.count(0) self._threads = {} @@ -1384,7 +1392,7 @@ def call(ctx: CephadmContext, desc: Optional[str] = None, verbosity: CallVerbosity = CallVerbosity.VERBOSE_ON_FAILURE, timeout: Optional[int] = DEFAULT_TIMEOUT, - **kwargs) -> Tuple[str, str, int]: + **kwargs: Any) -> Tuple[str, str, int]: """ Wrap subprocess.Popen to @@ -1417,7 +1425,8 @@ def call(ctx: CephadmContext, process = await asyncio.create_subprocess_exec( *command, stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE) + stderr=asyncio.subprocess.PIPE, + env=os.environ.copy()) assert process.stdout assert process.stderr try: @@ -1447,7 +1456,7 @@ def call_throws( desc: Optional[str] = None, verbosity: CallVerbosity = CallVerbosity.VERBOSE_ON_FAILURE, timeout: Optional[int] = DEFAULT_TIMEOUT, - **kwargs) -> Tuple[str, str, int]: + **kwargs: Any) -> Tuple[str, str, int]: out, err, ret = call(ctx, command, desc, verbosity, timeout, **kwargs) if ret: raise RuntimeError('Failed command: %s' % ' '.join(command)) @@ -1466,14 +1475,14 @@ def call_timeout(ctx, command, timeout): raise TimeoutExpired(msg) try: - return subprocess.call(command, timeout=timeout) + return subprocess.call(command, timeout=timeout, env=os.environ.copy()) except subprocess.TimeoutExpired: raise_timeout(command, timeout) ################################## -def json_loads_retry(cli_func): +def json_loads_retry(cli_func: Callable[[], str]) -> Any: for sleep_secs in [1, 4, 4]: try: return json.loads(cli_func()) @@ -1578,7 +1587,7 @@ def try_convert_datetime(s): def _parse_podman_version(version_str): # type: (str) -> Tuple[int, ...] - def to_int(val, org_e=None): + def to_int(val: str, org_e: Optional[Exception] = None) -> int: if not val and org_e: raise org_e try: @@ -1642,17 +1651,33 @@ def is_fsid(s): return True -def infer_fsid(func): +def validate_fsid(func: FuncT) -> FuncT: + @wraps(func) + def _validate_fsid(ctx: CephadmContext) -> Any: + if 'fsid' in ctx and ctx.fsid: + if not is_fsid(ctx.fsid): + raise Error('not an fsid: %s' % ctx.fsid) + return func(ctx) + return cast(FuncT, _validate_fsid) + + +def infer_fsid(func: FuncT) -> FuncT: """ If we only find a single fsid in /var/lib/ceph/*, use that """ + @infer_config @wraps(func) - def _infer_fsid(ctx: CephadmContext): - if ctx.fsid: + def _infer_fsid(ctx: CephadmContext) -> Any: + if 'fsid' in ctx and ctx.fsid: logger.debug('Using specified fsid: %s' % ctx.fsid) return func(ctx) - fsids_set = set() + fsids = set() + + cp = read_config(ctx.config) + if cp.has_option('global', 'fsid'): + fsids.add(cp.get('global', 'fsid')) + daemon_list = list_daemons(ctx, detail=False) for daemon in daemon_list: if not is_fsid(daemon['fsid']): @@ -1660,11 +1685,11 @@ def infer_fsid(func): continue elif 'name' not in ctx or not ctx.name: # ctx.name not specified - fsids_set.add(daemon['fsid']) + fsids.add(daemon['fsid']) elif daemon['name'] == ctx.name: # ctx.name is a match - fsids_set.add(daemon['fsid']) - fsids = sorted(fsids_set) + fsids.add(daemon['fsid']) + fsids = sorted(fsids) if not fsids: # some commands do not always require an fsid @@ -1676,42 +1701,40 @@ def infer_fsid(func): raise Error('Cannot infer an fsid, one must be specified: %s' % fsids) return func(ctx) - return _infer_fsid + return cast(FuncT, _infer_fsid) -def infer_config(func): +def infer_config(func: FuncT) -> FuncT: """ If we find a MON daemon, use the config from that container """ @wraps(func) - def _infer_config(ctx: CephadmContext): + def _infer_config(ctx: CephadmContext) -> Any: + ctx.config = ctx.config if 'config' in ctx else None if ctx.config: logger.debug('Using specified config: %s' % ctx.config) return func(ctx) - config = None - if ctx.fsid: - name = ctx.name + if 'fsid' in ctx and ctx.fsid: + name = ctx.name if 'name' in ctx else None if not name: daemon_list = list_daemons(ctx, detail=False) for daemon in daemon_list: - if daemon['name'].startswith('mon.'): + if daemon.get('name', '').startswith('mon.'): name = daemon['name'] break if name: - config = '/var/lib/ceph/{}/{}/config'.format(ctx.fsid, - name) - if config: - logger.info('Inferring config %s' % config) - ctx.config = config + ctx.config = f'/var/lib/ceph/{ctx.fsid}/{name}/config' + if ctx.config: + logger.info('Inferring config %s' % ctx.config) elif os.path.exists(SHELL_DEFAULT_CONF): logger.debug('Using default config: %s' % SHELL_DEFAULT_CONF) ctx.config = SHELL_DEFAULT_CONF return func(ctx) - return _infer_config + return cast(FuncT, _infer_config) -def _get_default_image(ctx: CephadmContext): +def _get_default_image(ctx: CephadmContext) -> str: if DEFAULT_IMAGE_IS_MASTER: warn = """This is a development version of cephadm. For information regarding the latest stable release: @@ -1722,12 +1745,12 @@ For information regarding the latest stable release: return DEFAULT_IMAGE -def infer_image(func): +def infer_image(func: FuncT) -> FuncT: """ Use the most recent ceph image """ @wraps(func) - def _infer_image(ctx: CephadmContext): + def _infer_image(ctx: CephadmContext) -> Any: if not ctx.image: ctx.image = os.environ.get('CEPHADM_IMAGE') if not ctx.image: @@ -1736,12 +1759,12 @@ def infer_image(func): ctx.image = _get_default_image(ctx) return func(ctx) - return _infer_image + return cast(FuncT, _infer_image) -def default_image(func): +def default_image(func: FuncT) -> FuncT: @wraps(func) - def _default_image(ctx: CephadmContext): + def _default_image(ctx: CephadmContext) -> Any: if not ctx.image: if 'name' in ctx and ctx.name: type_ = ctx.name.split('.', 1)[0] @@ -1758,10 +1781,10 @@ def default_image(func): return func(ctx) - return _default_image + return cast(FuncT, _default_image) -def get_last_local_ceph_image(ctx: CephadmContext, container_path: str): +def get_last_local_ceph_image(ctx: CephadmContext, container_path: str) -> Optional[str]: """ :return: The most recent local ceph image (already pulled) """ @@ -1920,7 +1943,7 @@ def move_files(ctx, src, dst, uid=None, gid=None): # copied from distutils -def find_executable(executable, path=None): +def find_executable(executable: str, path: Optional[str] = None) -> Optional[str]: """Tries to find 'executable' in the directories listed in 'path'. A string listing directories separated by 'os.pathsep'; defaults to os.environ['PATH']. Returns the complete filename or None if not found. @@ -1964,7 +1987,7 @@ def find_program(filename): return name -def find_container_engine(ctx: CephadmContext): +def find_container_engine(ctx: CephadmContext) -> Optional[ContainerEngine]: if ctx.docker: return Docker() else: @@ -1980,7 +2003,9 @@ def check_container_engine(ctx): # type: (CephadmContext) -> None engine = ctx.container_engine if not isinstance(engine, CONTAINER_PREFERENCE): - raise Error('Unable to locate any of %s' % [i.EXE for i in CONTAINER_PREFERENCE]) + # See https://github.com/python/mypy/issues/8993 + exes: List[str] = [i.EXE for i in CONTAINER_PREFERENCE] # type: ignore + raise Error('No container engine binary found ({}). Try run `apt/dnf/yum/zypper install `'.format(' or '.join(exes))) elif isinstance(engine, Podman): engine.get_version(ctx) if engine.version < MIN_PODMAN_VERSION: @@ -1998,7 +2023,7 @@ def get_unit_name(fsid, daemon_type, daemon_id=None): return 'ceph-%s@%s' % (fsid, daemon_type) -def get_unit_name_by_daemon_name(ctx: CephadmContext, fsid, name): +def get_unit_name_by_daemon_name(ctx: CephadmContext, fsid: str, name: str) -> str: daemon = get_daemon_description(ctx, fsid, name) try: return daemon['systemd_unit'] @@ -2059,12 +2084,19 @@ def check_units(ctx, units, enabler=None): return False -def is_container_running(ctx: CephadmContext, name: str) -> bool: - out, err, ret = call(ctx, [ - ctx.container_engine.path, 'container', 'inspect', - '--format', '{{.State.Status}}', name - ]) - return out == 'running' +def is_container_running(ctx: CephadmContext, c: 'CephContainer') -> bool: + return bool(get_running_container_name(ctx, c)) + + +def get_running_container_name(ctx: CephadmContext, c: 'CephContainer') -> Optional[str]: + for name in [c.cname, c.old_cname]: + out, err, ret = call(ctx, [ + ctx.container_engine.path, 'container', 'inspect', + '--format', '{{.State.Status}}', name + ]) + if out.strip() == 'running': + return name + return None def get_legacy_config_fsid(cluster, legacy_dir=None): @@ -2173,7 +2205,9 @@ def create_daemon_dirs(ctx, fsid, daemon_type, daemon_id, uid, gid, f.write(keyring) if daemon_type in Monitoring.components.keys(): - config_json: Dict[str, Any] = get_parm(ctx.config_json) + config_json: Dict[str, Any] = dict() + if 'config_json' in ctx: + config_json = get_parm(ctx.config_json) # Set up directories specific to the monitoring component config_dir = '' @@ -2361,6 +2395,7 @@ def get_container_mounts(ctx, fsid, daemon_type, daemon_id, ceph_folder = pathify(ctx.shared_ceph_folder) if os.path.exists(ceph_folder): mounts[ceph_folder + '/src/ceph-volume/ceph_volume'] = '/usr/lib/python3.6/site-packages/ceph_volume' + mounts[ceph_folder + '/src/cephadm/cephadm'] = '/usr/sbin/cephadm' mounts[ceph_folder + '/src/pybind/mgr'] = '/usr/share/ceph/mgr' mounts[ceph_folder + '/src/python-common/ceph'] = '/usr/lib/python3.6/site-packages/ceph' mounts[ceph_folder + '/monitoring/grafana/dashboards'] = '/etc/grafana/dashboards/ceph-dashboard' @@ -2428,11 +2463,11 @@ def get_container(ctx: CephadmContext, entrypoint: str = '' name: str = '' ceph_args: List[str] = [] - envs: List[str] = [ - 'TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728', - ] + envs: List[str] = [] host_network: bool = True + if daemon_type in Ceph.daemons: + envs.append('TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728') if container_args is None: container_args = [] if daemon_type in ['mon', 'osd']: @@ -2461,6 +2496,7 @@ def get_container(ctx: CephadmContext, envs.extend(NFSGanesha.get_container_envs()) elif daemon_type == HAproxy.daemon_type: name = '%s.%s' % (daemon_type, daemon_id) + container_args.extend(['--user=root']) # haproxy 2.4 defaults to a different user elif daemon_type == Keepalived.daemon_type: name = '%s.%s' % (daemon_type, daemon_id) envs.extend(Keepalived.get_container_envs()) @@ -2506,15 +2542,16 @@ def get_container(ctx: CephadmContext, if ctx.container_engine.version >= CGROUPS_SPLIT_PODMAN_VERSION: container_args.append('--cgroups=split') - return CephContainer( + return CephContainer.for_daemon( ctx, - image=ctx.image, + fsid=fsid, + daemon_type=daemon_type, + daemon_id=str(daemon_id), entrypoint=entrypoint, args=ceph_args + get_daemon_args(ctx, fsid, daemon_type, daemon_id), container_args=container_args, volume_mounts=get_container_mounts(ctx, fsid, daemon_type, daemon_id), bind_mounts=get_container_binds(ctx, fsid, daemon_type, daemon_id), - cname='ceph-%s-%s.%s' % (fsid, daemon_type, daemon_id), envs=envs, privileged=privileged, ptrace=ptrace, @@ -2623,6 +2660,7 @@ def deploy_daemon(ctx, fsid, daemon_type, daemon_id, c, uid, gid, else: config_js = get_parm(ctx.config_json) assert isinstance(config_js, dict) + assert isinstance(daemon_id, str) cephadm_exporter = CephadmDaemon(ctx, fsid, daemon_id, port) cephadm_exporter.deploy_daemon_unit(config_js) @@ -2668,6 +2706,7 @@ def _write_container_cmd_to_bash(ctx, file_obj, container, comment=None, backgro # unit file, makes it easier to read and grok. file_obj.write('# ' + comment + '\n') # Sometimes, adding `--rm` to a run_cmd doesn't work. Let's remove the container manually + file_obj.write('! ' + ' '.join(container.rm_cmd(old_cname=True)) + ' 2> /dev/null\n') file_obj.write('! ' + ' '.join(container.rm_cmd()) + ' 2> /dev/null\n') # Sometimes, `podman rm` doesn't find the container. Then you'll have to add `--storage` if isinstance(ctx.container_engine, Podman): @@ -2675,6 +2714,10 @@ def _write_container_cmd_to_bash(ctx, file_obj, container, comment=None, backgro '! ' + ' '.join([shlex.quote(a) for a in container.rm_cmd(storage=True)]) + ' 2> /dev/null\n') + file_obj.write( + '! ' + + ' '.join([shlex.quote(a) for a in container.rm_cmd(old_cname=True, storage=True)]) + + ' 2> /dev/null\n') # container run command file_obj.write( @@ -2682,6 +2725,31 @@ def _write_container_cmd_to_bash(ctx, file_obj, container, comment=None, backgro + (' &' if background else '') + '\n') +def clean_cgroup(ctx: CephadmContext, fsid: str, unit_name: str) -> None: + # systemd may fail to cleanup cgroups from previous stopped unit, which will cause next "systemctl start" to fail. + # see https://tracker.ceph.com/issues/50998 + + CGROUPV2_PATH = Path('/sys/fs/cgroup') + if not (CGROUPV2_PATH / 'system.slice').exists(): + # Only unified cgroup is affected, skip if not the case + return + + slice_name = 'system-ceph\\x2d{}.slice'.format(fsid.replace('-', '\\x2d')) + cg_path = CGROUPV2_PATH / 'system.slice' / slice_name / f'{unit_name}.service' + if not cg_path.exists(): + return + + def cg_trim(path: Path) -> None: + for p in path.iterdir(): + if p.is_dir(): + cg_trim(p) + path.rmdir() + try: + cg_trim(cg_path) + except OSError: + logger.warning(f'Failed to trim old cgroups {cg_path}') + + def deploy_daemon_units( ctx: CephadmContext, fsid: str, @@ -2790,6 +2858,15 @@ def deploy_daemon_units( os.rename(data_dir + '/unit.poststop.new', data_dir + '/unit.poststop') + # post-stop command(s) + with open(data_dir + '/unit.stop.new', 'w') as f: + f.write('! ' + ' '.join(c.stop_cmd()) + '\n') + f.write('! ' + ' '.join(c.stop_cmd(old_cname=True)) + '\n') + + os.fchmod(f.fileno(), 0o600) + os.rename(data_dir + '/unit.stop.new', + data_dir + '/unit.stop') + if c: with open(data_dir + '/unit.image.new', 'w') as f: f.write(c.image + '\n') @@ -2818,6 +2895,7 @@ def deploy_daemon_units( if enable: call_throws(ctx, ['systemctl', 'enable', unit_name]) if start: + clean_cgroup(ctx, fsid, unit_name) call_throws(ctx, ['systemctl', 'start', unit_name]) @@ -2960,6 +3038,7 @@ def install_sysctl(ctx: CephadmContext, fsid: str, daemon_type: str) -> None: # apply the sysctl settings if lines: + Path(ctx.sysctl_dir).mkdir(mode=0o755, exist_ok=True) _write(conf, lines) call_throws(ctx, ['sysctl', '--system']) @@ -3066,7 +3145,7 @@ LimitNOFILE=1048576 LimitNPROC=1048576 EnvironmentFile=-/etc/environment ExecStart=/bin/bash {data_dir}/{fsid}/%i/unit.run -ExecStop=-{container_path} stop ceph-{fsid}-%i +ExecStop=-/bin/bash -c '{container_path} stop ceph-{fsid}-%i ; bash {data_dir}/{fsid}/%i/unit.stop' ExecStopPost=-/bin/bash {data_dir}/{fsid}/%i/unit.poststop KillMode=none Restart=on-failure @@ -3114,7 +3193,7 @@ class CephContainer: self.entrypoint = entrypoint self.args = args self.volume_mounts = volume_mounts - self.cname = cname + self._cname = cname self.container_args = container_args self.envs = envs self.privileged = privileged @@ -3125,6 +3204,73 @@ class CephContainer: self.memory_request = memory_request self.memory_limit = memory_limit + @classmethod + def for_daemon(cls, + ctx: CephadmContext, + fsid: str, + daemon_type: str, + daemon_id: str, + entrypoint: str, + args: List[str] = [], + volume_mounts: Dict[str, str] = {}, + container_args: List[str] = [], + envs: Optional[List[str]] = None, + privileged: bool = False, + ptrace: bool = False, + bind_mounts: Optional[List[List[str]]] = None, + init: Optional[bool] = None, + host_network: bool = True, + memory_request: Optional[str] = None, + memory_limit: Optional[str] = None, + ) -> 'CephContainer': + return cls( + ctx, + image=ctx.image, + entrypoint=entrypoint, + args=args, + volume_mounts=volume_mounts, + cname='ceph-%s-%s.%s' % (fsid, daemon_type, daemon_id), + container_args=container_args, + envs=envs, + privileged=privileged, + ptrace=ptrace, + bind_mounts=bind_mounts, + init=init, + host_network=host_network, + memory_request=memory_request, + memory_limit=memory_limit, + ) + + @property + def cname(self) -> str: + """ + podman adds the current container name to the /etc/hosts + file. Turns out, python's `socket.getfqdn()` differs from + `hostname -f`, when we have the container names containing + dots in it.: + + # podman run --name foo.bar.baz.com ceph/ceph /bin/bash + [root@sebastians-laptop /]# cat /etc/hosts + 127.0.0.1 localhost + ::1 localhost + 127.0.1.1 sebastians-laptop foo.bar.baz.com + [root@sebastians-laptop /]# hostname -f + sebastians-laptop + [root@sebastians-laptop /]# python3 -c 'import socket; print(socket.getfqdn())' + foo.bar.baz.com + + Fascinatingly, this doesn't happen when using dashes. + """ + return self._cname.replace('.', '-') + + @cname.setter + def cname(self, val: str) -> None: + self._cname = val + + @property + def old_cname(self) -> str: + return self._cname + def run_cmd(self) -> List[str]: cmd_args: List[str] = [ str(self.ctx.container_engine.path), @@ -3232,6 +3378,9 @@ class CephContainer: def exec_cmd(self, cmd): # type: (List[str]) -> List[str] + cname = get_running_container_name(self.ctx, self) + if not cname: + raise Error('unable to find container "{}"'.format(self.cname)) return [ str(self.ctx.container_engine.path), 'exec', @@ -3239,22 +3388,23 @@ class CephContainer: self.cname, ] + cmd - def rm_cmd(self, storage=False): - # type: (bool) -> List[str] + def rm_cmd(self, old_cname: bool = False, storage: bool = False) -> List[str]: ret = [ str(self.ctx.container_engine.path), 'rm', '-f', ] if storage: ret.append('--storage') - ret.append(self.cname) + if old_cname: + ret.append(self.old_cname) + else: + ret.append(self.cname) return ret - def stop_cmd(self): - # type () -> List[str] + def stop_cmd(self, old_cname: bool = False) -> List[str]: ret = [ str(self.ctx.container_engine.path), - 'stop', self.cname, + 'stop', self.old_cname if old_cname else self.cname, ] return ret @@ -3336,7 +3486,7 @@ def command_inspect_image(ctx): return 0 -def normalize_image_digest(digest): +def normalize_image_digest(digest: str) -> str: # normal case: # ceph/ceph -> docker.io/ceph/ceph # edge cases that shouldn't ever come up: @@ -3437,7 +3587,8 @@ def prepare_mon_addresses( ctx.mon_ip = wrap_ipv6(ctx.mon_ip) hasport = r.findall(ctx.mon_ip) if hasport: - port = int(hasport[0]) + port_str = hasport[0] + port = int(port_str) if port == 6789: addr_arg = '[v1:%s]' % ctx.mon_ip elif port == 3300: @@ -3446,7 +3597,7 @@ def prepare_mon_addresses( logger.warning('Using msgr2 protocol for unrecognized port %d' % port) addr_arg = '[v2:%s]' % ctx.mon_ip - base_ip = ctx.mon_ip[0:-(len(str(port))) - 1] + base_ip = ctx.mon_ip[0:-(len(port_str)) - 1] check_ip_port(ctx, base_ip, port) else: base_ip = ctx.mon_ip @@ -3464,10 +3615,11 @@ def prepare_mon_addresses( if not hasport: raise Error('--mon-addrv value %s must include port number' % addr_arg) - port = int(hasport[0]) + port_str = hasport[0] + port = int(port_str) # strip off v1: or v2: prefix - addr = re.sub(r'^\w+:', '', addr) - base_ip = addr[0:-(len(str(port))) - 1] + addr = re.sub(r'^v\d+:', '', addr) + base_ip = addr[0:-(len(port_str)) - 1] check_ip_port(ctx, base_ip, port) else: raise Error('must specify --mon-ip or --mon-addrv') @@ -3606,7 +3758,7 @@ def prepare_create_mon( fsid: str, mon_id: str, bootstrap_keyring_path: str, monmap_path: str -): +) -> Tuple[str, str]: logger.info('Creating mon...') create_daemon_dirs(ctx, fsid, 'mon', mon_id, uid, gid) mon_dir = get_data_dir(fsid, ctx.data_dir, 'mon', mon_id) @@ -3649,7 +3801,7 @@ def wait_for_mon( ctx: CephadmContext, mon_id: str, mon_dir: str, admin_keyring_path: str, config_path: str -): +) -> None: logger.info('Waiting for mon to start...') c = CephContainer( ctx, @@ -3772,7 +3924,7 @@ def prepare_ssh( try: args = ['orch', 'host', 'add', host] if ctx.mon_ip: - args.append(ctx.mon_ip) + args.append(unwrap_ipv6(ctx.mon_ip)) cli(args) except RuntimeError as e: raise Error('Failed to add host <%s>: %s' % (host, e)) @@ -3790,8 +3942,6 @@ def prepare_ssh( cli(['orch', 'apply', 'crash']) if not ctx.skip_monitoring_stack: - logger.info('Enabling mgr prometheus module...') - cli(['mgr', 'module', 'enable', 'prometheus']) for t in ['prometheus', 'grafana', 'node-exporter', 'alertmanager']: logger.info('Deploying %s service with default placement...' % t) cli(['orch', 'apply', t]) @@ -3904,6 +4054,13 @@ def prepare_bootstrap_config( and not cp.has_option('mgr', 'mgr standby modules') ): cp.set('mgr', 'mgr_standby_modules', 'false') + if ctx.log_to_file: + cp.set('global', 'log_to_file', 'true') + cp.set('global', 'log_to_stderr', 'false') + cp.set('global', 'log_to_journald', 'false') + cp.set('global', 'mon_cluster_log_to_file', 'true') + cp.set('global', 'mon_cluster_log_to_stderr', 'false') + cp.set('global', 'mon_cluster_log_to_journald', 'false') cpf = StringIO() cp.write(cpf) @@ -4111,7 +4268,7 @@ def command_bootstrap(ctx): {tmp.name: '/var/lib/ceph/user.conf:z'}) # wait for mgr to restart (after enabling a module) - def wait_for_mgr_restart(): + def wait_for_mgr_restart() -> None: # first get latest mgrmap epoch from the mon. try newer 'mgr # stat' command first, then fall back to 'mgr dump' if # necessary @@ -4218,7 +4375,7 @@ def command_bootstrap(ctx): ################################## -def command_registry_login(ctx: CephadmContext): +def command_registry_login(ctx: CephadmContext) -> int: if ctx.registry_json: logger.info('Pulling custom registry login info from %s.' % ctx.registry_json) d = get_parm(ctx.registry_json) @@ -4244,7 +4401,7 @@ def command_registry_login(ctx: CephadmContext): return 0 -def registry_login(ctx: CephadmContext, url, username, password): +def registry_login(ctx: CephadmContext, url: Optional[str], username: Optional[str], password: Optional[str]) -> None: logger.info('Logging into custom registry.') try: engine = ctx.container_engine @@ -4291,9 +4448,8 @@ def command_deploy(ctx): redeploy = False unit_name = get_unit_name(ctx.fsid, daemon_type, daemon_id) - container_name = 'ceph-%s-%s.%s' % (ctx.fsid, daemon_type, daemon_id) (_, state, _) = check_unit(ctx, unit_name) - if state == 'running' or is_container_running(ctx, container_name): + if state == 'running' or is_container_running(ctx, CephContainer.for_daemon(ctx, ctx.fsid, daemon_type, daemon_id, 'bash')): redeploy = True if ctx.reconfig: @@ -4429,24 +4585,16 @@ def command_run(ctx): ################################## -def fsid_conf_mismatch(ctx): - # type: (CephadmContext) -> bool - (config, _) = get_config_and_keyring(ctx) - if config: - for c in config.split('\n'): - if 'fsid = ' in c.strip(): - if 'fsid = ' + ctx.fsid != c.strip(): - return True - return False - - @infer_fsid @infer_config @infer_image +@validate_fsid def command_shell(ctx): # type: (CephadmContext) -> int - if fsid_conf_mismatch(ctx): - raise Error('fsid does not match ceph conf') + cp = read_config(ctx.config) + if cp.has_option('global', 'fsid') and \ + cp.get('global', 'fsid') != ctx.fsid: + raise Error('fsid does not match ceph.conf') if ctx.fsid: make_log_dir(ctx, ctx.fsid) @@ -4560,8 +4708,14 @@ def command_enter(ctx): @infer_fsid @infer_image +@validate_fsid def command_ceph_volume(ctx): # type: (CephadmContext) -> None + cp = read_config(ctx.config) + if cp.has_option('global', 'fsid') and \ + cp.get('global', 'fsid') != ctx.fsid: + raise Error('fsid does not match ceph.conf') + if ctx.fsid: make_log_dir(ctx, ctx.fsid) @@ -4638,13 +4792,13 @@ def command_logs(ctx): # call this directly, without our wrapper, so that we get an unmolested # stdout with logger prefixing. logger.debug('Running command: %s' % ' '.join(cmd)) - subprocess.call(cmd) # type: ignore + subprocess.call(cmd, env=os.environ.copy()) # type: ignore ################################## def list_networks(ctx): - # type: (CephadmContext) -> Dict[str,Dict[str,List[str]]] + # type: (CephadmContext) -> Dict[str,Dict[str, Set[str]]] # sadly, 18.04's iproute2 4.15.0-2ubun doesn't support the -j flag, # so we'll need to use a regex to parse 'ip' command output. @@ -4652,13 +4806,12 @@ def list_networks(ctx): # out, _, _ = call_throws(['ip', '-j', 'route', 'ls']) # j = json.loads(out) # for x in j: - res = _list_ipv4_networks(ctx) res.update(_list_ipv6_networks(ctx)) return res -def _list_ipv4_networks(ctx: CephadmContext): +def _list_ipv4_networks(ctx: CephadmContext) -> Dict[str, Dict[str, Set[str]]]: execstr: Optional[str] = find_executable('ip') if not execstr: raise FileNotFoundError("unable to find 'ip' command") @@ -4666,8 +4819,8 @@ def _list_ipv4_networks(ctx: CephadmContext): return _parse_ipv4_route(out) -def _parse_ipv4_route(out): - r = {} # type: Dict[str,Dict[str,List[str]]] +def _parse_ipv4_route(out: str) -> Dict[str, Dict[str, Set[str]]]: + r = {} # type: Dict[str, Dict[str, Set[str]]] p = re.compile(r'^(\S+) dev (\S+) (.*)scope link (.*)src (\S+)') for line in out.splitlines(): m = p.findall(line) @@ -4679,12 +4832,12 @@ def _parse_ipv4_route(out): if net not in r: r[net] = {} if iface not in r[net]: - r[net][iface] = [] - r[net][iface].append(ip) + r[net][iface] = set() + r[net][iface].add(ip) return r -def _list_ipv6_networks(ctx: CephadmContext): +def _list_ipv6_networks(ctx: CephadmContext) -> Dict[str, Dict[str, Set[str]]]: execstr: Optional[str] = find_executable('ip') if not execstr: raise FileNotFoundError("unable to find 'ip' command") @@ -4693,8 +4846,8 @@ def _list_ipv6_networks(ctx: CephadmContext): return _parse_ipv6_route(routes, ips) -def _parse_ipv6_route(routes, ips): - r = {} # type: Dict[str,Dict[str,List[str]]] +def _parse_ipv6_route(routes: str, ips: str) -> Dict[str, Dict[str, Set[str]]]: + r = {} # type: Dict[str, Dict[str, Set[str]]] route_p = re.compile(r'^(\S+) dev (\S+) proto (\S+) metric (\S+) .*pref (\S+)$') ip_p = re.compile(r'^\s+inet6 (\S+)/(.*)scope (.*)$') iface_p = re.compile(r'^(\d+): (\S+): (.*)$') @@ -4709,7 +4862,7 @@ def _parse_ipv6_route(routes, ips): if net not in r: r[net] = {} if iface not in r[net]: - r[net][iface] = [] + r[net][iface] = set() iface = None for line in ips.splitlines(): @@ -4726,7 +4879,7 @@ def _parse_ipv6_route(routes, ips): if ipaddress.ip_address(ip) in ipaddress.ip_network(n)] if net: assert(iface) - r[net[0]][iface].append(ip) + r[net[0]][iface].add(ip) return r @@ -4734,7 +4887,11 @@ def _parse_ipv6_route(routes, ips): def command_list_networks(ctx): # type: (CephadmContext) -> None r = list_networks(ctx) - print(json.dumps(r, indent=4)) + + def serialize_sets(obj: Any) -> Any: + return list(obj) if isinstance(obj, set) else obj + + print(json.dumps(r, indent=4, default=serialize_sets)) ################################## @@ -4790,14 +4947,7 @@ def list_daemons(ctx, detail=True, legacy_dir=None): [container_path, 'stats', '--format', '{{.ID}},{{.MemUsage}}', '--no-stream'], verbosity=CallVerbosity.DEBUG ) - seen_memusage_cid_len = 0 - if not code: - for line in out.splitlines(): - (cid, usage) = line.split(',') - (used, limit) = usage.split(' / ') - seen_memusage[cid] = with_units_to_int(used) - if not seen_memusage_cid_len: - seen_memusage_cid_len = len(cid) + seen_memusage_cid_len, seen_memusage = _parse_mem_usage(code, out) # /var/lib/ceph if os.path.exists(data_dir): @@ -4860,12 +5010,7 @@ def list_daemons(ctx, detail=True, legacy_dir=None): version = None start_stamp = None - cmd = [ - container_path, 'inspect', - '--format', '{{.Id}},{{.Config.Image}},{{.Image}},{{.Created}},{{index .Config.Labels "io.ceph.version"}}', - 'ceph-%s-%s' % (fsid, j) - ] - out, err, code = call(ctx, cmd, verbosity=CallVerbosity.DEBUG) + out, err, code = get_container_stats(ctx, container_path, fsid, daemon_type, daemon_id) if not code: (container_id, image_name, image_id, start, version) = out.strip().split(',') @@ -4985,6 +5130,24 @@ def list_daemons(ctx, detail=True, legacy_dir=None): return ls +def _parse_mem_usage(code: int, out: str) -> Tuple[int, Dict[str, int]]: + # keep track of memory usage we've seen + seen_memusage = {} # type: Dict[str, int] + seen_memusage_cid_len = 0 + if not code: + for line in out.splitlines(): + (cid, usage) = line.split(',') + (used, limit) = usage.split(' / ') + try: + seen_memusage[cid] = with_units_to_int(used) + if not seen_memusage_cid_len: + seen_memusage_cid_len = len(cid) + except ValueError: + logger.info('unable to parse memory usage line\n>{}'.format(line)) + pass + return seen_memusage_cid_len, seen_memusage + + def get_daemon_description(ctx, fsid, name, detail=False, legacy_dir=None): # type: (CephadmContext, str, str, bool, Optional[str]) -> Dict[str, str] @@ -4996,6 +5159,21 @@ def get_daemon_description(ctx, fsid, name, detail=False, legacy_dir=None): return d raise Error('Daemon not found: {}. See `cephadm ls`'.format(name)) + +def get_container_stats(ctx: CephadmContext, container_path: str, fsid: str, daemon_type: str, daemon_id: str) -> Tuple[str, str, int]: + c = CephContainer.for_daemon(ctx, fsid, daemon_type, daemon_id, 'bash') + out, err, code = '', '', -1 + for name in (c.cname, c.old_cname): + cmd = [ + container_path, 'inspect', + '--format', '{{.Id}},{{.Config.Image}},{{.Image}},{{.Created}},{{index .Config.Labels "io.ceph.version"}}', + name + ] + out, err, code = call(ctx, cmd, verbosity=CallVerbosity.DEBUG) + if not code: + break + return out, err, code + ################################## @@ -5427,7 +5605,7 @@ def command_rm_daemon(ctx): ################################## -def _zap(ctx, what): +def _zap(ctx: CephadmContext, what: str) -> None: mounts = get_container_mounts(ctx, ctx.fsid, 'clusterless-ceph-volume', None) c = CephContainer( ctx, @@ -5443,7 +5621,7 @@ def _zap(ctx, what): @infer_image -def _zap_osds(ctx): +def _zap_osds(ctx: CephadmContext) -> None: # assume fsid lock already held # list @@ -5476,7 +5654,7 @@ def _zap_osds(ctx): logger.warning(f'Not zapping LVs (not implemented): {lv_names}') -def command_zap_osds(ctx): +def command_zap_osds(ctx: CephadmContext) -> None: if not ctx.force: raise Error('must pass --force to proceed: ' 'this command may destroy precious data!') @@ -5521,7 +5699,7 @@ def command_rm_cluster(ctx): call(ctx, ['systemctl', 'disable', unit_name], verbosity=CallVerbosity.DEBUG) - slice_name = 'system-%s.slice' % (('ceph-%s' % ctx.fsid).replace('-', '\\x2d')) + slice_name = 'system-ceph\\x2d{}.slice'.format(ctx.fsid.replace('-', '\\x2d')) call(ctx, ['systemctl', 'stop', slice_name], verbosity=CallVerbosity.DEBUG) @@ -5583,6 +5761,7 @@ def check_time_sync(ctx, enabler=None): 'ntpd.service', # el7 (at least) 'ntp.service', # 18.04 (at least) 'ntpsec.service', # 20.04 (at least) / buster + 'openntpd.service', # ubuntu / debian ] if not check_units(ctx, units, enabler): logger.warning('No time sync service is running; checked for %s' % units) @@ -5668,7 +5847,7 @@ def command_prepare_host(ctx: CephadmContext) -> None: class CustomValidation(argparse.Action): - def _check_name(self, values): + def _check_name(self, values: str) -> None: try: (daemon_type, daemon_id) = values.split('.', 1) except ValueError: @@ -5681,7 +5860,9 @@ class CustomValidation(argparse.Action): 'name must declare the type of daemon e.g. ' '{}'.format(', '.join(daemons))) - def __call__(self, parser, namespace, values, option_string=None): + def __call__(self, parser: argparse.ArgumentParser, namespace: argparse.Namespace, values: Union[str, Sequence[Any], None], + option_string: Optional[str] = None) -> None: + assert isinstance(values, str) if self.dest == 'name': self._check_name(values) setattr(namespace, self.dest, values) @@ -5723,7 +5904,8 @@ def get_distro(): class Packager(object): def __init__(self, ctx: CephadmContext, - stable=None, version=None, branch=None, commit=None): + stable: Optional[str] = None, version: Optional[str] = None, + branch: Optional[str] = None, commit: Optional[str] = None): assert \ (stable and not version and not branch and not commit) or \ (not stable and version and not branch and not commit) or \ @@ -5735,13 +5917,19 @@ class Packager(object): self.branch = branch self.commit = commit - def add_repo(self): + def add_repo(self) -> None: + raise NotImplementedError + + def rm_repo(self) -> None: + raise NotImplementedError + + def install(self, ls: List[str]) -> None: raise NotImplementedError - def rm_repo(self): + def install_podman(self) -> None: raise NotImplementedError - def query_shaman(self, distro, distro_version, branch, commit): + def query_shaman(self, distro: str, distro_version: Any, branch: Optional[str], commit: Optional[str]) -> str: # query shaman logger.info('Fetching repo metadata from shaman and chacra...') shaman_url = 'https://shaman.ceph.com/api/repos/ceph/{branch}/{sha1}/{distro}/{distro_version}/repo/?arch={arch}'.format( @@ -5765,7 +5953,7 @@ class Packager(object): raise Error('%s, failed to fetch %s' % (err, chacra_url)) return chacra_response.read().decode('utf-8') - def repo_gpgkey(self): + def repo_gpgkey(self) -> Tuple[str, str]: if self.ctx.gpg_url: return self.ctx.gpg_url if self.stable or self.version: @@ -5773,7 +5961,7 @@ class Packager(object): else: return 'https://download.ceph.com/keys/autobuild.gpg', 'autobuild' - def enable_service(self, service): + def enable_service(self, service: str) -> None: """ Start and enable the service (typically using systemd). """ @@ -5787,19 +5975,20 @@ class Apt(Packager): } def __init__(self, ctx: CephadmContext, - stable, version, branch, commit, - distro, distro_version, distro_codename): + stable: Optional[str], version: Optional[str], branch: Optional[str], commit: Optional[str], + distro: Optional[str], distro_version: Optional[str], distro_codename: Optional[str]) -> None: super(Apt, self).__init__(ctx, stable=stable, version=version, branch=branch, commit=commit) + assert distro self.ctx = ctx self.distro = self.DISTRO_NAMES[distro] self.distro_codename = distro_codename self.distro_version = distro_version - def repo_path(self): + def repo_path(self) -> str: return '/etc/apt/sources.list.d/ceph.list' - def add_repo(self): + def add_repo(self) -> None: url, name = self.repo_gpgkey() logger.info('Installing repo GPG key from %s...' % url) @@ -5829,7 +6018,7 @@ class Apt(Packager): self.update() - def rm_repo(self): + def rm_repo(self) -> None: for name in ['autobuild', 'release']: p = '/etc/apt/trusted.gpg.d/ceph.%s.gpg' % name if os.path.exists(p): @@ -5842,15 +6031,15 @@ class Apt(Packager): if self.distro == 'ubuntu': self.rm_kubic_repo() - def install(self, ls): + def install(self, ls: List[str]) -> None: logger.info('Installing packages %s...' % ls) call_throws(self.ctx, ['apt-get', 'install', '-y'] + ls) - def update(self): + def update(self) -> None: logger.info('Updating package list...') call_throws(self.ctx, ['apt-get', 'update']) - def install_podman(self): + def install_podman(self) -> None: if self.distro == 'ubuntu': logger.info('Setting up repo for podman...') self.add_kubic_repo() @@ -5863,20 +6052,20 @@ class Apt(Packager): logger.info('Podman did not work. Falling back to docker...') self.install(['docker.io']) - def kubic_repo_url(self): + def kubic_repo_url(self) -> str: return 'https://download.opensuse.org/repositories/devel:/kubic:/' \ 'libcontainers:/stable/xUbuntu_%s/' % self.distro_version - def kubic_repo_path(self): + def kubic_repo_path(self) -> str: return '/etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list' - def kubric_repo_gpgkey_url(self): + def kubric_repo_gpgkey_url(self) -> str: return '%s/Release.key' % self.kubic_repo_url() - def kubric_repo_gpgkey_path(self): + def kubric_repo_gpgkey_path(self) -> str: return '/etc/apt/trusted.gpg.d/kubic.release.gpg' - def add_kubic_repo(self): + def add_kubic_repo(self) -> None: url = self.kubric_repo_gpgkey_url() logger.info('Installing repo GPG key from %s...' % url) try: @@ -5895,7 +6084,7 @@ class Apt(Packager): with open(self.kubic_repo_path(), 'w') as f: f.write(content) - def rm_kubic_repo(self): + def rm_kubic_repo(self) -> None: keyring = self.kubric_repo_gpgkey_path() if os.path.exists(keyring): logger.info('Removing repo GPG key %s...' % keyring) @@ -5913,14 +6102,17 @@ class YumDnf(Packager): 'rhel': ('centos', 'el'), 'scientific': ('centos', 'el'), 'rocky': ('centos', 'el'), + 'almalinux': ('centos', 'el'), 'fedora': ('fedora', 'fc'), } def __init__(self, ctx: CephadmContext, - stable, version, branch, commit, - distro, distro_version): + stable: Optional[str], version: Optional[str], branch: Optional[str], commit: Optional[str], + distro: Optional[str], distro_version: Optional[str]) -> None: super(YumDnf, self).__init__(ctx, stable=stable, version=version, branch=branch, commit=commit) + assert distro + assert distro_version self.ctx = ctx self.major = int(distro_version.split('.')[0]) self.distro_normalized = self.DISTRO_NAMES[distro][0] @@ -5931,7 +6123,7 @@ class YumDnf(Packager): else: self.tool = 'yum' - def custom_repo(self, **kw): + def custom_repo(self, **kw: Any) -> str: """ Repo files need special care in that a whole line should not be present if there is no value for it. Because we were using `format()` we could @@ -5987,10 +6179,10 @@ class YumDnf(Packager): return '\n'.join(lines) - def repo_path(self): + def repo_path(self) -> str: return '/etc/yum.repos.d/ceph.repo' - def repo_baseurl(self): + def repo_baseurl(self) -> str: assert self.stable or self.version if self.version: return '%s/rpm-%s/%s' % (self.ctx.repo_url, self.version, @@ -5999,7 +6191,7 @@ class YumDnf(Packager): return '%s/rpm-%s/%s' % (self.ctx.repo_url, self.stable, self.distro_code) - def add_repo(self): + def add_repo(self) -> None: if self.distro_code.startswith('fc'): raise Error('Ceph team does not build Fedora specific packages and therefore cannot add repos for this distro') if self.distro_code == 'el7': @@ -6035,15 +6227,15 @@ class YumDnf(Packager): logger.info('Enabling EPEL...') call_throws(self.ctx, [self.tool, 'install', '-y', 'epel-release']) - def rm_repo(self): + def rm_repo(self) -> None: if os.path.exists(self.repo_path()): os.unlink(self.repo_path()) - def install(self, ls): + def install(self, ls: List[str]) -> None: logger.info('Installing packages %s...' % ls) call_throws(self.ctx, [self.tool, 'install', '-y'] + ls) - def install_podman(self): + def install_podman(self) -> None: self.install(['podman']) @@ -6055,10 +6247,11 @@ class Zypper(Packager): ] def __init__(self, ctx: CephadmContext, - stable, version, branch, commit, - distro, distro_version): + stable: Optional[str], version: Optional[str], branch: Optional[str], commit: Optional[str], + distro: Optional[str], distro_version: Optional[str]) -> None: super(Zypper, self).__init__(ctx, stable=stable, version=version, branch=branch, commit=commit) + assert distro is not None self.ctx = ctx self.tool = 'zypper' self.distro = 'opensuse' @@ -6066,7 +6259,7 @@ class Zypper(Packager): if 'tumbleweed' not in distro and distro_version is not None: self.distro_version = distro_version - def custom_repo(self, **kw): + def custom_repo(self, **kw: Any) -> str: """ See YumDnf for format explanation. """ @@ -6095,10 +6288,10 @@ class Zypper(Packager): return '\n'.join(lines) - def repo_path(self): + def repo_path(self) -> str: return '/etc/zypp/repos.d/ceph.repo' - def repo_baseurl(self): + def repo_baseurl(self) -> str: assert self.stable or self.version if self.version: return '%s/rpm-%s/%s' % (self.ctx.repo_url, @@ -6107,7 +6300,7 @@ class Zypper(Packager): return '%s/rpm-%s/%s' % (self.ctx.repo_url, self.stable, self.distro) - def add_repo(self): + def add_repo(self) -> None: if self.stable or self.version: content = '' for n, t in { @@ -6132,20 +6325,21 @@ class Zypper(Packager): with open(self.repo_path(), 'w') as f: f.write(content) - def rm_repo(self): + def rm_repo(self) -> None: if os.path.exists(self.repo_path()): os.unlink(self.repo_path()) - def install(self, ls): + def install(self, ls: List[str]) -> None: logger.info('Installing packages %s...' % ls) call_throws(self.ctx, [self.tool, 'in', '-y'] + ls) - def install_podman(self): + def install_podman(self) -> None: self.install(['podman']) def create_packager(ctx: CephadmContext, - stable=None, version=None, branch=None, commit=None): + stable: Optional[str] = None, version: Optional[str] = None, + branch: Optional[str] = None, commit: Optional[str] = None) -> Packager: distro, distro_version, distro_codename = get_distro() if distro in YumDnf.DISTRO_NAMES: return YumDnf(ctx, stable=stable, version=version, @@ -6163,7 +6357,7 @@ def create_packager(ctx: CephadmContext, raise Error('Distro %s version %s not supported' % (distro, distro_version)) -def command_add_repo(ctx: CephadmContext): +def command_add_repo(ctx: CephadmContext) -> None: if ctx.version and ctx.release: raise Error('you can specify either --release or --version but not both') if not ctx.version and not ctx.release and not ctx.dev and not ctx.dev_commit: @@ -6185,12 +6379,12 @@ def command_add_repo(ctx: CephadmContext): logger.info('Completed adding repo.') -def command_rm_repo(ctx: CephadmContext): +def command_rm_repo(ctx: CephadmContext) -> None: pkg = create_packager(ctx) pkg.rm_repo() -def command_install(ctx: CephadmContext): +def command_install(ctx: CephadmContext) -> None: pkg = create_packager(ctx) pkg.install(ctx.packages) @@ -6199,7 +6393,7 @@ def command_install(ctx: CephadmContext): def get_ipv4_address(ifname): # type: (str) -> str - def _extract(sock, offset): + def _extract(sock: socket.socket, offset: int) -> str: return socket.inet_ntop( socket.AF_INET, fcntl.ioctl( @@ -6296,7 +6490,6 @@ def read_file(path_list, file_name=''): class HostFacts(): _dmi_path_list = ['/sys/class/dmi/id'] _nic_path_list = ['/sys/class/net'] - _selinux_path_list = ['/etc/selinux/config'] _apparmor_path_list = ['/etc/apparmor'] _disk_vendor_workarounds = { '0x1af4': 'Virtio Block Device' @@ -6653,23 +6846,30 @@ class HostFacts(): # type: () -> Dict[str, str] """Determine the security features enabled in the kernel - SELinux, AppArmor""" def _fetch_selinux() -> Dict[str, str]: - """Read the selinux config file to determine state""" + """Get the selinux status""" security = {} - for selinux_path in HostFacts._selinux_path_list: - if os.path.exists(selinux_path): - selinux_config = read_file([selinux_path]).splitlines() - security['type'] = 'SELinux' - for line in selinux_config: - if line.strip().startswith('#'): - continue - k, v = line.split('=') - security[k] = v - if security['SELINUX'].lower() == 'disabled': - security['description'] = 'SELinux: Disabled' - else: - security['description'] = 'SELinux: Enabled({}, {})'.format(security['SELINUX'], security['SELINUXTYPE']) - return security - return {} + try: + out, err, code = call(self.ctx, ['sestatus'], + verbosity=CallVerbosity.DEBUG) + security['type'] = 'SELinux' + status, mode, policy = '', '', '' + for line in out.split('\n'): + if line.startswith('SELinux status:'): + k, v = line.split(':') + status = v.strip() + elif line.startswith('Current mode:'): + k, v = line.split(':') + mode = v.strip() + elif line.startswith('Loaded policy name:'): + k, v = line.split(':') + policy = v.strip() + if status == 'disabled': + security['description'] = 'SELinux: Disabled' + else: + security['description'] = 'SELinux: Enabled({}, {})'.format(mode, policy) + except Exception as e: + logger.info('unable to get selinux status: %s' % e) + return security def _fetch_apparmor() -> Dict[str, str]: """Read the apparmor profiles directly, returning an overview of AppArmor status""" @@ -6722,7 +6922,7 @@ class HostFacts(): } @property - def selinux_enabled(self): + def selinux_enabled(self) -> bool: return (self.kernel_security['type'] == 'SELinux') and \ (self.kernel_security['description'] != 'SELinux: Disabled') @@ -6743,6 +6943,48 @@ class HostFacts(): return k_param + @staticmethod + def _process_net_data(tcp_file: str, protocol: str = 'tcp') -> List[int]: + listening_ports = [] + # Connections state documentation + # tcp - https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/net/tcp_states.h + # udp - uses 07 (TCP_CLOSE or UNCONN, since udp is stateless. test with netcat -ul ) + listening_state = { + 'tcp': '0A', + 'udp': '07' + } + + if protocol not in listening_state.keys(): + return [] + + if os.path.exists(tcp_file): + with open(tcp_file) as f: + tcp_data = f.readlines()[1:] + + for con in tcp_data: + con_info = con.strip().split() + if con_info[3] == listening_state[protocol]: + local_port = int(con_info[1].split(':')[1], 16) + listening_ports.append(local_port) + + return listening_ports + + @property + def tcp_ports_used(self) -> List[int]: + return HostFacts._process_net_data('/proc/net/tcp') + + @property + def tcp6_ports_used(self) -> List[int]: + return HostFacts._process_net_data('/proc/net/tcp6') + + @property + def udp_ports_used(self) -> List[int]: + return HostFacts._process_net_data('/proc/net/udp', 'udp') + + @property + def udp6_ports_used(self) -> List[int]: + return HostFacts._process_net_data('/proc/net/udp6', 'udp') + def dump(self): # type: () -> str """Return the attributes of this HostFacts object as json""" @@ -6756,7 +6998,7 @@ class HostFacts(): ################################## -def command_gather_facts(ctx: CephadmContext): +def command_gather_facts(ctx: CephadmContext) -> None: """gather_facts is intended to provide host releated metadata to the caller""" host = HostFacts(ctx) print(host.dump()) @@ -6768,7 +7010,7 @@ def command_gather_facts(ctx: CephadmContext): class CephadmCache: task_types = ['disks', 'daemons', 'host', 'http_server'] - def __init__(self): + def __init__(self) -> None: self.started_epoch_secs = time.time() self.tasks = { 'daemons': 'inactive', @@ -6776,21 +7018,21 @@ class CephadmCache: 'host': 'inactive', 'http_server': 'inactive', } - self.errors = [] - self.disks = {} - self.daemons = {} - self.host = {} + self.errors: list = [] + self.disks: dict = {} + self.daemons: dict = {} + self.host: dict = {} self.lock = RLock() @property - def health(self): + def health(self) -> dict: return { 'started_epoch_secs': self.started_epoch_secs, 'tasks': self.tasks, 'errors': self.errors, } - def to_json(self): + def to_json(self) -> dict: return { 'health': self.health, 'host': self.host, @@ -6798,14 +7040,14 @@ class CephadmCache: 'disks': self.disks, } - def update_health(self, task_type, task_status, error_msg=None): + def update_health(self, task_type: str, task_status: str, error_msg: Optional[str] = None) -> None: assert task_type in CephadmCache.task_types with self.lock: self.tasks[task_type] = task_status if error_msg: self.errors.append(error_msg) - def update_task(self, task_type, content): + def update_task(self, task_type: str, content: dict) -> None: assert task_type in CephadmCache.task_types assert isinstance(content, dict) with self.lock: @@ -6836,14 +7078,14 @@ class CephadmDaemonHandler(BaseHTTPRequestHandler): class Decorators: @classmethod - def authorize(cls, f): + def authorize(cls, f: Any) -> Any: """Implement a basic token check. The token is installed at deployment time and must be provided to ensure we only respond to callers who know our token i.e. mgr """ - def wrapper(self, *args, **kwargs): + def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: auth = self.headers.get('Authorization', None) if auth != 'Bearer ' + self.server.token: self.send_error(401) @@ -6852,7 +7094,7 @@ class CephadmDaemonHandler(BaseHTTPRequestHandler): return wrapper - def _help_page(self): + def _help_page(self) -> str: return """ cephadm metadata exporter @@ -6888,14 +7130,14 @@ td,th {{ """.format(api_version=CephadmDaemonHandler.api_version) - def _fetch_root(self): + def _fetch_root(self) -> None: self.send_response(200) self.send_header('Content-type', 'text/html; charset=utf-8') self.end_headers() self.wfile.write(self._help_page().encode('utf-8')) @Decorators.authorize - def do_GET(self): + def do_GET(self) -> None: """Handle *all* GET requests""" if self.path == '/': @@ -6953,7 +7195,7 @@ td,th {{ self.end_headers() self.wfile.write(json.dumps({'message': bad_request_msg}).encode('utf-8')) - def log_message(self, format, *args): + def log_message(self, format: str, *args: Any) -> None: rqst = ' '.join(str(a) for a in args) logger.info(f'client:{self.address_string()} [{self.log_date_time_string()}] {rqst}') @@ -6973,7 +7215,7 @@ class CephadmDaemon(): loop_delay = 1 thread_check_interval = 5 - def __init__(self, ctx: CephadmContext, fsid, daemon_id=None, port=None): + def __init__(self, ctx: CephadmContext, fsid: str, daemon_id: Optional[str] = None, port: Optional[int] = None) -> None: self.ctx = ctx self.fsid = fsid self.daemon_id = daemon_id @@ -6989,7 +7231,7 @@ class CephadmDaemon(): self.token = read_file([os.path.join(self.daemon_path, CephadmDaemon.token_name)]) @classmethod - def validate_config(cls, config): + def validate_config(cls, config: dict) -> None: reqs = ', '.join(CephadmDaemon.config_requirements) errors = [] @@ -7022,11 +7264,11 @@ class CephadmDaemon(): raise Error('Parameter errors : {}'.format(', '.join(errors))) @property - def port_active(self): + def port_active(self) -> bool: return port_in_use(self.ctx, self.port) @property - def can_run(self): + def can_run(self) -> bool: # if port is in use if self.port_active: self.errors.append(f'TCP port {self.port} already in use, unable to bind') @@ -7039,15 +7281,16 @@ class CephadmDaemon(): return len(self.errors) == 0 @staticmethod - def _unit_name(fsid, daemon_id): + def _unit_name(fsid: str, daemon_id: str) -> str: return '{}.service'.format(get_unit_name(fsid, CephadmDaemon.daemon_type, daemon_id)) @property - def unit_name(self): + def unit_name(self) -> str: + assert self.daemon_id is not None return CephadmDaemon._unit_name(self.fsid, self.daemon_id) @property - def daemon_path(self): + def daemon_path(self) -> str: return os.path.join( self.ctx.data_dir, self.fsid, @@ -7055,12 +7298,12 @@ class CephadmDaemon(): ) @property - def binary_path(self): + def binary_path(self) -> str: path = os.path.realpath(__file__) assert os.path.isfile(path) return path - def _handle_thread_exception(self, exc, thread_type): + def _handle_thread_exception(self, exc: Exception, thread_type: str) -> None: e_msg = f'{exc.__class__.__name__} exception: {str(exc)}' thread_info = getattr(self.cephadm_cache, thread_type) errors = thread_info.get('scrape_errors', []) @@ -7075,7 +7318,7 @@ class CephadmDaemon(): } ) - def _scrape_host_facts(self, refresh_interval=10): + def _scrape_host_facts(self, refresh_interval: int = 10) -> None: ctr = 0 exception_encountered = False @@ -7118,7 +7361,7 @@ class CephadmDaemon(): ctr += CephadmDaemon.loop_delay logger.info('host-facts thread stopped') - def _scrape_ceph_volume(self, refresh_interval=15): + def _scrape_ceph_volume(self, refresh_interval: int = 15) -> None: # we're invoking the ceph_volume command, so we need to set the args that it # expects to use self.ctx.command = 'inventory --format=json'.split() @@ -7177,7 +7420,7 @@ class CephadmDaemon(): logger.info('ceph-volume thread stopped') - def _scrape_list_daemons(self, refresh_interval=20): + def _scrape_list_daemons(self, refresh_interval: int = 20) -> None: ctr = 0 exception_encountered = False while True: @@ -7217,7 +7460,7 @@ class CephadmDaemon(): ctr += CephadmDaemon.loop_delay logger.info('list-daemons thread stopped') - def _create_thread(self, target, name, refresh_interval=None): + def _create_thread(self, target: Any, name: str, refresh_interval: Optional[int] = None) -> Thread: if refresh_interval: t = Thread(target=target, args=(refresh_interval,)) else: @@ -7234,7 +7477,7 @@ class CephadmDaemon(): logger.info(f'{start_msg}') return t - def reload(self, *args): + def reload(self, *args: Any) -> None: """reload -HUP received This is a placeholder function only, and serves to provide the hook that could @@ -7242,12 +7485,12 @@ class CephadmDaemon(): """ logger.info('Reload request received - ignoring, no action needed') - def shutdown(self, *args): + def shutdown(self, *args: Any) -> None: logger.info('Shutdown request received') self.stop = True self.http_server.shutdown() - def run(self): + def run(self) -> None: logger.info(f"cephadm exporter starting for FSID '{self.fsid}'") if not self.can_run: logger.error('Unable to start the exporter daemon') @@ -7302,7 +7545,7 @@ class CephadmDaemon(): logger.info('Main http server thread stopped') @property - def unit_run(self): + def unit_run(self) -> str: return """set -e {py3} {bin_path} exporter --fsid {fsid} --id {daemon_id} --port {port} &""".format( @@ -7314,7 +7557,7 @@ class CephadmDaemon(): ) @property - def unit_file(self): + def unit_file(self) -> str: docker = isinstance(self.ctx.container_engine, Docker) return """#generated by cephadm [Unit] @@ -7341,7 +7584,7 @@ WantedBy=ceph-{fsid}.target docker_after=' docker.service' if docker else '', docker_requires='Requires=docker.service\n' if docker else '') - def deploy_daemon_unit(self, config=None): + def deploy_daemon_unit(self, config: Optional[dict] = None) -> None: """deploy a specific unit file for cephadm The normal deploy_daemon_units doesn't apply for this @@ -7361,8 +7604,11 @@ WantedBy=ceph-{fsid}.target # we pick up the file from where the orchestrator placed it - otherwise we'll # copy it to the binary location for this cluster if not __file__ == '': - shutil.copy(__file__, - self.binary_path) + try: + shutil.copy(__file__, + self.binary_path) + except shutil.SameFileError: + pass with open(os.path.join(self.daemon_path, 'unit.run'), 'w') as f: f.write(self.unit_run) @@ -7385,7 +7631,7 @@ WantedBy=ceph-{fsid}.target call_throws(self.ctx, ['systemctl', 'enable', '--now', self.unit_name]) @classmethod - def uninstall(cls, ctx: CephadmContext, fsid, daemon_type, daemon_id): + def uninstall(cls, ctx: CephadmContext, fsid: str, daemon_type: str, daemon_id: str) -> None: unit_name = CephadmDaemon._unit_name(fsid, daemon_id) unit_path = os.path.join(ctx.unit_dir, unit_name) unit_run = os.path.join(ctx.data_dir, fsid, f'{daemon_type}.{daemon_id}', 'unit.run') @@ -7423,7 +7669,7 @@ WantedBy=ceph-{fsid}.target stdout, stderr, rc = call(ctx, ['systemctl', 'daemon-reload']) -def command_exporter(ctx: CephadmContext): +def command_exporter(ctx: CephadmContext) -> None: exporter = CephadmDaemon(ctx, ctx.fsid, daemon_id=ctx.id, port=ctx.port) if ctx.fsid not in os.listdir(ctx.data_dir): @@ -7446,7 +7692,7 @@ def systemd_target_state(target_name: str, subsystem: str = 'ceph') -> bool: @infer_fsid -def command_maintenance(ctx: CephadmContext): +def command_maintenance(ctx: CephadmContext) -> str: if not ctx.fsid: raise Error('must pass --fsid to specify cluster') @@ -7495,6 +7741,7 @@ def command_maintenance(ctx: CephadmContext): return 'failed - unable to start the target' else: return f'success - systemd target {target} enabled and started' + return f'success - systemd target {target} enabled and started' ################################## @@ -7874,7 +8121,6 @@ def _get_parser(): '--ssh-user', default='root', help='set user for SSHing to cluster hosts, passwordless sudo will be needed for non-root users') - parser_bootstrap.add_argument( '--skip-mon-network', action='store_true', @@ -7930,7 +8176,6 @@ def _get_parser(): parser_bootstrap.add_argument( '--apply-spec', help='Apply cluster spec after bootstrap (copy ssh key, add hosts and apply services)') - parser_bootstrap.add_argument( '--shared_ceph_folder', metavar='CEPH_SOURCE_FOLDER', @@ -7968,6 +8213,10 @@ def _get_parser(): '--single-host-defaults', action='store_true', help='adjust configuration defaults to suit a single-host cluster') + parser_bootstrap.add_argument( + '--log-to-file', + action='store_true', + help='configure cluster to log to traditional log files in /var/log/ceph/$fsid') parser_deploy = subparsers.add_parser( 'deploy', help='deploy a daemon') @@ -8136,7 +8385,7 @@ def _get_parser(): return parser -def _parse_args(av): +def _parse_args(av: List[str]) -> argparse.Namespace: parser = _get_parser() args = parser.parse_args(av) @@ -8193,7 +8442,7 @@ def cephadm_init(args: List[str]) -> CephadmContext: return ctx -def main(): +def main() -> None: # root? if os.geteuid() != 0: diff --git a/ceph/src/cephadm/tests/fixtures.py b/ceph/src/cephadm/tests/fixtures.py index 2f07d6034..ed1e73766 100644 --- a/ceph/src/cephadm/tests/fixtures.py +++ b/ceph/src/cephadm/tests/fixtures.py @@ -69,6 +69,8 @@ def cephadm_fs( gid = os.getgid() with mock.patch('os.fchown'), \ + mock.patch('os.fchmod'), \ + mock.patch('platform.processor', return_value='x86_64'), \ mock.patch('cephadm.extract_uid_gid', return_value=(uid, gid)): fs.create_dir(cd.DATA_DIR) @@ -93,19 +95,21 @@ def with_cephadm_ctx( :param list_networks: mock 'list-networks' return :param hostname: mock 'socket.gethostname' return """ - if not list_networks: - list_networks = {} if not hostname: hostname = 'host1' - with mock.patch('cephadm.get_parm'), \ - mock.patch('cephadm.attempt_bind'), \ + with mock.patch('cephadm.attempt_bind'), \ mock.patch('cephadm.call', return_value=('', '', 0)), \ + mock.patch('cephadm.call_timeout', return_value=0), \ mock.patch('cephadm.find_executable', return_value='foo'), \ mock.patch('cephadm.is_available', return_value=True), \ mock.patch('cephadm.json_loads_retry', return_value={'epoch' : 1}), \ - mock.patch('cephadm.list_networks', return_value=list_networks), \ mock.patch('socket.gethostname', return_value=hostname): - ctx: cd.CephadmContext = cd.cephadm_init_ctx(cmd) - ctx.container_engine = container_engine - yield ctx + ctx: cd.CephadmContext = cd.cephadm_init_ctx(cmd) + ctx.container_engine = container_engine + if list_networks is not None: + with mock.patch('cephadm.list_networks', return_value=list_networks): + yield ctx + else: + yield ctx + diff --git a/ceph/src/cephadm/tests/test_cephadm.py b/ceph/src/cephadm/tests/test_cephadm.py index 42853fb6b..40afdd422 100644 --- a/ceph/src/cephadm/tests/test_cephadm.py +++ b/ceph/src/cephadm/tests/test_cephadm.py @@ -1,6 +1,7 @@ # type: ignore import errno +import json import mock import os import pytest @@ -30,10 +31,20 @@ with mock.patch('builtins.open', create=True): cd = SourceFileLoader('cephadm', 'cephadm').load_module() +def get_ceph_conf( + fsid='00000000-0000-0000-0000-0000deadbeef', + mon_host='[v2:192.168.1.1:3300/0,v1:192.168.1.1:6789/0]'): + return f''' +# minimal ceph.conf for {fsid} +[global] + fsid = {fsid} + mon_host = {mon_host} +''' + class TestCephAdm(object): def test_docker_unit_file(self): - ctx = mock.Mock() + ctx = cd.CephadmContext() ctx.container_engine = mock_docker() r = cd.get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9') assert 'Requires=docker.service' in r @@ -91,7 +102,7 @@ class TestCephAdm(object): @mock.patch('socket.socket') @mock.patch('cephadm.logger') def test_check_ip_port_success(self, logger, _socket): - ctx = mock.Mock() + ctx = cd.CephadmContext() ctx.skip_ping_check = False # enables executing port check with `check_ip_port` for address, address_family in ( @@ -108,7 +119,7 @@ class TestCephAdm(object): @mock.patch('socket.socket') @mock.patch('cephadm.logger') def test_check_ip_port_failure(self, logger, _socket): - ctx = mock.Mock() + ctx = cd.CephadmContext() ctx.skip_ping_check = False # enables executing port check with `check_ip_port` def os_error(errno): @@ -148,6 +159,11 @@ class TestCephAdm(object): args = cd._parse_args(['--image', 'foo', 'version']) assert args.image == 'foo' + def test_parse_mem_usage(self): + cd.logger = mock.Mock() + len, summary = cd._parse_mem_usage(0, 'c6290e3f1489,-- / --') + assert summary == {} + def test_CustomValidation(self): assert cd._parse_args(['deploy', '--name', 'mon.a', '--fsid', 'fsid']) @@ -166,204 +182,6 @@ class TestCephAdm(object): cd._parse_podman_version('inval.id') assert 'inval' in str(res.value) - @pytest.mark.parametrize("test_input, expected", [ - ( -""" -default via 192.168.178.1 dev enxd89ef3f34260 proto dhcp metric 100 -10.0.0.0/8 via 10.4.0.1 dev tun0 proto static metric 50 -10.3.0.0/21 via 10.4.0.1 dev tun0 proto static metric 50 -10.4.0.1 dev tun0 proto kernel scope link src 10.4.0.2 metric 50 -137.1.0.0/16 via 10.4.0.1 dev tun0 proto static metric 50 -138.1.0.0/16 via 10.4.0.1 dev tun0 proto static metric 50 -139.1.0.0/16 via 10.4.0.1 dev tun0 proto static metric 50 -140.1.0.0/17 via 10.4.0.1 dev tun0 proto static metric 50 -141.1.0.0/16 via 10.4.0.1 dev tun0 proto static metric 50 -169.254.0.0/16 dev docker0 scope link metric 1000 -172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 -192.168.39.0/24 dev virbr1 proto kernel scope link src 192.168.39.1 linkdown -192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 linkdown -192.168.178.0/24 dev enxd89ef3f34260 proto kernel scope link src 192.168.178.28 metric 100 -192.168.178.1 dev enxd89ef3f34260 proto static scope link metric 100 -195.135.221.12 via 192.168.178.1 dev enxd89ef3f34260 proto static metric 100 -""", - { - '10.4.0.1': {'tun0': ['10.4.0.2']}, - '172.17.0.0/16': {'docker0': ['172.17.0.1']}, - '192.168.39.0/24': {'virbr1': ['192.168.39.1']}, - '192.168.122.0/24': {'virbr0': ['192.168.122.1']}, - '192.168.178.0/24': {'enxd89ef3f34260': ['192.168.178.28']} - } - ), ( -""" -default via 10.3.64.1 dev eno1 proto static metric 100 -10.3.64.0/24 dev eno1 proto kernel scope link src 10.3.64.23 metric 100 -10.3.64.0/24 dev eno1 proto kernel scope link src 10.3.64.27 metric 100 -10.88.0.0/16 dev cni-podman0 proto kernel scope link src 10.88.0.1 linkdown -172.21.0.0/20 via 172.21.3.189 dev tun0 -172.21.1.0/20 via 172.21.3.189 dev tun0 -172.21.2.1 via 172.21.3.189 dev tun0 -172.21.3.1 dev tun0 proto kernel scope link src 172.21.3.2 -172.21.4.0/24 via 172.21.3.1 dev tun0 -172.21.5.0/24 via 172.21.3.1 dev tun0 -172.21.6.0/24 via 172.21.3.1 dev tun0 -172.21.7.0/24 via 172.21.3.1 dev tun0 -192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 linkdown -""", - { - '10.3.64.0/24': {'eno1': ['10.3.64.23', '10.3.64.27']}, - '10.88.0.0/16': {'cni-podman0': ['10.88.0.1']}, - '172.21.3.1': {'tun0': ['172.21.3.2']}, - '192.168.122.0/24': {'virbr0': ['192.168.122.1']}} - ), - ]) - def test_parse_ipv4_route(self, test_input, expected): - assert cd._parse_ipv4_route(test_input) == expected - - @pytest.mark.parametrize("test_routes, test_ips, expected", [ - ( -""" -::1 dev lo proto kernel metric 256 pref medium -fe80::/64 dev eno1 proto kernel metric 100 pref medium -fe80::/64 dev br-3d443496454c proto kernel metric 256 linkdown pref medium -fe80::/64 dev tun0 proto kernel metric 256 pref medium -fe80::/64 dev br-4355f5dbb528 proto kernel metric 256 pref medium -fe80::/64 dev docker0 proto kernel metric 256 linkdown pref medium -fe80::/64 dev cni-podman0 proto kernel metric 256 linkdown pref medium -fe80::/64 dev veth88ba1e8 proto kernel metric 256 pref medium -fe80::/64 dev vethb6e5fc7 proto kernel metric 256 pref medium -fe80::/64 dev vethaddb245 proto kernel metric 256 pref medium -fe80::/64 dev vethbd14d6b proto kernel metric 256 pref medium -fe80::/64 dev veth13e8fd2 proto kernel metric 256 pref medium -fe80::/64 dev veth1d3aa9e proto kernel metric 256 pref medium -fe80::/64 dev vethe485ca9 proto kernel metric 256 pref medium -""", -""" -1: lo: mtu 65536 state UNKNOWN qlen 1000 - inet6 ::1/128 scope host - valid_lft forever preferred_lft forever -2: eno1: mtu 1500 state UP qlen 1000 - inet6 fe80::225:90ff:fee5:26e8/64 scope link noprefixroute - valid_lft forever preferred_lft forever -6: br-3d443496454c: mtu 1500 state DOWN - inet6 fe80::42:23ff:fe9d:ee4/64 scope link - valid_lft forever preferred_lft forever -7: br-4355f5dbb528: mtu 1500 state UP - inet6 fe80::42:6eff:fe35:41fe/64 scope link - valid_lft forever preferred_lft forever -8: docker0: mtu 1500 state DOWN - inet6 fe80::42:faff:fee6:40a0/64 scope link - valid_lft forever preferred_lft forever -11: tun0: mtu 1500 state UNKNOWN qlen 100 - inet6 fe80::98a6:733e:dafd:350/64 scope link stable-privacy - valid_lft forever preferred_lft forever -28: cni-podman0: mtu 1500 state DOWN qlen 1000 - inet6 fe80::3449:cbff:fe89:b87e/64 scope link - valid_lft forever preferred_lft forever -31: vethaddb245@if30: mtu 1500 state UP - inet6 fe80::90f7:3eff:feed:a6bb/64 scope link - valid_lft forever preferred_lft forever -33: veth88ba1e8@if32: mtu 1500 state UP - inet6 fe80::d:f5ff:fe73:8c82/64 scope link - valid_lft forever preferred_lft forever -35: vethbd14d6b@if34: mtu 1500 state UP - inet6 fe80::b44f:8ff:fe6f:813d/64 scope link - valid_lft forever preferred_lft forever -37: vethb6e5fc7@if36: mtu 1500 state UP - inet6 fe80::4869:c6ff:feaa:8afe/64 scope link - valid_lft forever preferred_lft forever -39: veth13e8fd2@if38: mtu 1500 state UP - inet6 fe80::78f4:71ff:fefe:eb40/64 scope link - valid_lft forever preferred_lft forever -41: veth1d3aa9e@if40: mtu 1500 state UP - inet6 fe80::24bd:88ff:fe28:5b18/64 scope link - valid_lft forever preferred_lft forever -43: vethe485ca9@if42: mtu 1500 state UP - inet6 fe80::6425:87ff:fe42:b9f0/64 scope link - valid_lft forever preferred_lft forever -""", - { - "fe80::/64": { - "eno1": [ - "fe80::225:90ff:fee5:26e8" - ], - "br-3d443496454c": [ - "fe80::42:23ff:fe9d:ee4" - ], - "tun0": [ - "fe80::98a6:733e:dafd:350" - ], - "br-4355f5dbb528": [ - "fe80::42:6eff:fe35:41fe" - ], - "docker0": [ - "fe80::42:faff:fee6:40a0" - ], - "cni-podman0": [ - "fe80::3449:cbff:fe89:b87e" - ], - "veth88ba1e8": [ - "fe80::d:f5ff:fe73:8c82" - ], - "vethb6e5fc7": [ - "fe80::4869:c6ff:feaa:8afe" - ], - "vethaddb245": [ - "fe80::90f7:3eff:feed:a6bb" - ], - "vethbd14d6b": [ - "fe80::b44f:8ff:fe6f:813d" - ], - "veth13e8fd2": [ - "fe80::78f4:71ff:fefe:eb40" - ], - "veth1d3aa9e": [ - "fe80::24bd:88ff:fe28:5b18" - ], - "vethe485ca9": [ - "fe80::6425:87ff:fe42:b9f0" - ] - } - } - ), - ( -""" -::1 dev lo proto kernel metric 256 pref medium -2001:1458:301:eb::100:1a dev ens20f0 proto kernel metric 100 pref medium -2001:1458:301:eb::/64 dev ens20f0 proto ra metric 100 pref medium -fd01:1458:304:5e::/64 dev ens20f0 proto ra metric 100 pref medium -fe80::/64 dev ens20f0 proto kernel metric 100 pref medium -default proto ra metric 100 - nexthop via fe80::46ec:ce00:b8a0:d3c8 dev ens20f0 weight 1 - nexthop via fe80::46ec:ce00:b8a2:33c8 dev ens20f0 weight 1 pref medium -""", -""" -1: lo: mtu 65536 state UNKNOWN qlen 1000 - inet6 ::1/128 scope host - valid_lft forever preferred_lft forever -2: ens20f0: mtu 1500 state UP qlen 1000 - inet6 2001:1458:301:eb::100:1a/128 scope global dynamic noprefixroute - valid_lft 590879sec preferred_lft 590879sec - inet6 fe80::2e60:cff:fef8:da41/64 scope link noprefixroute - valid_lft forever preferred_lft forever -""", - { - '2001:1458:301:eb::/64': { - 'ens20f0': [ - '2001:1458:301:eb::100:1a' - ], - }, - 'fe80::/64': { - 'ens20f0': ['fe80::2e60:cff:fef8:da41'], - }, - 'fd01:1458:304:5e::/64': { - 'ens20f0': [] - }, - } - ), - ]) - def test_parse_ipv6_route(self, test_routes, test_ips, expected): - assert cd._parse_ipv6_route(test_routes, test_ips) == expected - def test_is_ipv6(self): cd.logger = mock.Mock() for good in ("[::1]", "::1", @@ -522,6 +340,184 @@ docker.io/ceph/daemon-base:octopus s = 'ceph/ceph:latest' assert cd.normalize_image_digest(s) == f'{cd.DEFAULT_REGISTRY}/{s}' + @pytest.mark.parametrize('fsid, ceph_conf, list_daemons, result, err, ', + [ + ( + None, + None, + [], + None, + None, + ), + ( + '00000000-0000-0000-0000-0000deadbeef', + None, + [], + '00000000-0000-0000-0000-0000deadbeef', + None, + ), + ( + '00000000-0000-0000-0000-0000deadbeef', + None, + [ + {'fsid': '10000000-0000-0000-0000-0000deadbeef'}, + {'fsid': '20000000-0000-0000-0000-0000deadbeef'}, + ], + '00000000-0000-0000-0000-0000deadbeef', + None, + ), + ( + None, + None, + [ + {'fsid': '00000000-0000-0000-0000-0000deadbeef'}, + ], + '00000000-0000-0000-0000-0000deadbeef', + None, + ), + ( + None, + None, + [ + {'fsid': '10000000-0000-0000-0000-0000deadbeef'}, + {'fsid': '20000000-0000-0000-0000-0000deadbeef'}, + ], + None, + r'Cannot infer an fsid', + ), + ( + None, + get_ceph_conf(fsid='00000000-0000-0000-0000-0000deadbeef'), + [], + '00000000-0000-0000-0000-0000deadbeef', + None, + ), + ( + None, + get_ceph_conf(fsid='00000000-0000-0000-0000-0000deadbeef'), + [ + {'fsid': '00000000-0000-0000-0000-0000deadbeef'}, + ], + '00000000-0000-0000-0000-0000deadbeef', + None, + ), + ( + None, + get_ceph_conf(fsid='00000000-0000-0000-0000-0000deadbeef'), + [ + {'fsid': '10000000-0000-0000-0000-0000deadbeef'}, + {'fsid': '20000000-0000-0000-0000-0000deadbeef'}, + ], + None, + r'Cannot infer an fsid', + ), + ]) + @mock.patch('cephadm.call') + def test_infer_fsid(self, _call, fsid, ceph_conf, list_daemons, result, err, cephadm_fs): + # build the context + ctx = cd.CephadmContext() + ctx.fsid = fsid + + # mock the decorator + mock_fn = mock.Mock() + mock_fn.return_value = 0 + infer_fsid = cd.infer_fsid(mock_fn) + + # mock the ceph.conf file content + if ceph_conf: + f = cephadm_fs.create_file('ceph.conf', contents=ceph_conf) + ctx.config = f.path + + # test + with mock.patch('cephadm.list_daemons', return_value=list_daemons): + if err: + with pytest.raises(cd.Error, match=err): + infer_fsid(ctx) + else: + infer_fsid(ctx) + assert ctx.fsid == result + + @pytest.mark.parametrize('fsid, config, name, list_daemons, result, ', + [ + ( + None, + '/foo/bar.conf', + None, + [], + '/foo/bar.conf', + ), + ( + '00000000-0000-0000-0000-0000deadbeef', + None, + None, + [], + cd.SHELL_DEFAULT_CONF, + ), + ( + '00000000-0000-0000-0000-0000deadbeef', + None, + None, + [{'name': 'mon.a'}], + '/var/lib/ceph/00000000-0000-0000-0000-0000deadbeef/mon.a/config', + ), + ( + '00000000-0000-0000-0000-0000deadbeef', + None, + None, + [{'name': 'osd.0'}], + cd.SHELL_DEFAULT_CONF, + ), + ( + '00000000-0000-0000-0000-0000deadbeef', + '/foo/bar.conf', + 'mon.a', + [{'name': 'mon.a'}], + '/foo/bar.conf', + ), + ( + '00000000-0000-0000-0000-0000deadbeef', + None, + 'mon.a', + [], + '/var/lib/ceph/00000000-0000-0000-0000-0000deadbeef/mon.a/config', + ), + ( + '00000000-0000-0000-0000-0000deadbeef', + None, + 'osd.0', + [], + '/var/lib/ceph/00000000-0000-0000-0000-0000deadbeef/osd.0/config', + ), + ( + None, + None, + None, + [], + cd.SHELL_DEFAULT_CONF, + ), + ]) + @mock.patch('cephadm.call') + def test_infer_config(self, _call, fsid, config, name, list_daemons, result, cephadm_fs): + # build the context + ctx = cd.CephadmContext() + ctx.fsid = fsid + ctx.config = config + ctx.name = name + + # mock the decorator + mock_fn = mock.Mock() + mock_fn.return_value = 0 + infer_config = cd.infer_config(mock_fn) + + # mock the shell config + cephadm_fs.create_file(cd.SHELL_DEFAULT_CONF) + + # test + with mock.patch('cephadm.list_daemons', return_value=list_daemons): + infer_config(ctx) + assert ctx.config == result + + class TestCustomContainer(unittest.TestCase): cc: cd.CustomContainer @@ -925,7 +921,8 @@ class TestMaintenance: class TestMonitoring(object): @mock.patch('cephadm.call') def test_get_version_alertmanager(self, _call): - ctx = mock.Mock() + ctx = cd.CephadmContext() + ctx.container_engine = mock_podman() daemon_type = 'alertmanager' # binary `prometheus` @@ -943,7 +940,8 @@ class TestMonitoring(object): @mock.patch('cephadm.call') def test_get_version_prometheus(self, _call): - ctx = mock.Mock() + ctx = cd.CephadmContext() + ctx.container_engine = mock_podman() daemon_type = 'prometheus' _call.return_value = '', '{}, version 0.16.1'.format(daemon_type), 0 version = cd.Monitoring.get_version(ctx, 'container_id', daemon_type) @@ -951,20 +949,14 @@ class TestMonitoring(object): @mock.patch('cephadm.call') def test_get_version_node_exporter(self, _call): - ctx = mock.Mock() + ctx = cd.CephadmContext() + ctx.container_engine = mock_podman() daemon_type = 'node-exporter' _call.return_value = '', '{}, version 0.16.1'.format(daemon_type.replace('-', '_')), 0 version = cd.Monitoring.get_version(ctx, 'container_id', daemon_type) assert version == '0.16.1' - @mock.patch('cephadm.os.fchown') - @mock.patch('cephadm.get_parm') - @mock.patch('cephadm.makedirs') - @mock.patch('cephadm.open') - @mock.patch('cephadm.make_log_dir') - @mock.patch('cephadm.make_data_dir') - def test_create_daemon_dirs_prometheus(self, make_data_dir, make_log_dir, _open, makedirs, - get_parm, fchown): + def test_create_daemon_dirs_prometheus(self, cephadm_fs): """ Ensures the required and optional files given in the configuration are created and mapped correctly inside the container. Tests absolute and @@ -975,15 +967,14 @@ class TestMonitoring(object): daemon_type = 'prometheus' uid, gid = 50, 50 daemon_id = 'home' - ctx = mock.Mock() + ctx = cd.CephadmContext() ctx.data_dir = '/somedir' - files = { + ctx.config_json = json.dumps({ 'files': { 'prometheus.yml': 'foo', '/etc/prometheus/alerting/ceph_alerts.yml': 'bar' } - } - get_parm.return_value = files + }) cd.create_daemon_dirs(ctx, fsid, @@ -1000,14 +991,17 @@ class TestMonitoring(object): daemon_type=daemon_type, daemon_id=daemon_id ) - assert _open.call_args_list == [ - mock.call('{}/etc/prometheus/prometheus.yml'.format(prefix), 'w', - encoding='utf-8'), - mock.call('{}/etc/prometheus/alerting/ceph_alerts.yml'.format(prefix), 'w', - encoding='utf-8'), - ] - assert mock.call().__enter__().write('foo') in _open.mock_calls - assert mock.call().__enter__().write('bar') in _open.mock_calls + + expected = { + 'etc/prometheus/prometheus.yml': 'foo', + 'etc/prometheus/alerting/ceph_alerts.yml': 'bar', + } + + for file,content in expected.items(): + file = os.path.join(prefix, file) + assert os.path.exists(file) + with open(file) as f: + assert f.read() == content class TestBootstrap(object): @@ -1088,6 +1082,11 @@ class TestBootstrap(object): {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, True, ), + ( + '192.168.1.1:0123', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + True, + ), # IPv6 ( '::', @@ -1109,6 +1108,16 @@ class TestBootstrap(object): {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}}, True, ), + ( + '[::ffff:c0a8:101]:1234', + {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}}, + True, + ), + ( + '[::ffff:c0a8:101]:0123', + {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}}, + True, + ), ( '0000:0000:0000:0000:0000:FFFF:C0A8:0101', {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}}, @@ -1127,6 +1136,67 @@ class TestBootstrap(object): retval = cd.command_bootstrap(ctx) assert retval == 0 + @pytest.mark.parametrize('mon_addrv, list_networks, err', + [ + # IPv4 + ( + '192.168.1.1', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + r'must use square backets', + ), + ( + '[192.168.1.1]', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + r'must include port number', + ), + ( + '[192.168.1.1:1234]', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + None, + ), + ( + '[192.168.1.1:0123]', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + None, + ), + ( + '[v2:192.168.1.1:3300,v1:192.168.1.1:6789]', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + None, + ), + # IPv6 + ( + '[::ffff:192.168.1.1:1234]', + {'ffff::/64': {'eth0': ['::ffff:c0a8:101']}}, + None, + ), + ( + '[::ffff:192.168.1.1:0123]', + {'ffff::/64': {'eth0': ['::ffff:c0a8:101']}}, + None, + ), + ( + '[0000:0000:0000:0000:0000:FFFF:C0A8:0101:1234]', + {'ffff::/64': {'eth0': ['::ffff:c0a8:101']}}, + None, + ), + ( + '[v2:0000:0000:0000:0000:0000:FFFF:C0A8:0101:3300,v1:0000:0000:0000:0000:0000:FFFF:C0A8:0101:6789]', + {'ffff::/64': {'eth0': ['::ffff:c0a8:101']}}, + None, + ), + ]) + def test_mon_addrv(self, mon_addrv, list_networks, err, cephadm_fs): + cmd = self._get_cmd('--mon-addrv', mon_addrv) + if err: + with with_cephadm_ctx(cmd, list_networks=list_networks) as ctx: + with pytest.raises(cd.Error, match=err): + cd.command_bootstrap(ctx) + else: + with with_cephadm_ctx(cmd, list_networks=list_networks) as ctx: + retval = cd.command_bootstrap(ctx) + assert retval == 0 + def test_allow_fqdn_hostname(self, cephadm_fs): hostname = 'foo.bar' cmd = self._get_cmd( @@ -1164,3 +1234,230 @@ class TestBootstrap(object): else: retval = cd.command_bootstrap(ctx) assert retval == 0 + + +class TestShell(object): + + def test_fsid(self, cephadm_fs): + fsid = '00000000-0000-0000-0000-0000deadbeef' + + cmd = ['shell', '--fsid', fsid] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert ctx.fsid == fsid + + cmd = ['shell', '--fsid', '00000000-0000-0000-0000-0000deadbeez'] + with with_cephadm_ctx(cmd) as ctx: + err = 'not an fsid' + with pytest.raises(cd.Error, match=err): + retval = cd.command_shell(ctx) + assert retval == 1 + assert ctx.fsid == None + + s = get_ceph_conf(fsid=fsid) + f = cephadm_fs.create_file('ceph.conf', contents=s) + + cmd = ['shell', '--fsid', fsid, '--config', f.path] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert ctx.fsid == fsid + + cmd = ['shell', '--fsid', '10000000-0000-0000-0000-0000deadbeef', '--config', f.path] + with with_cephadm_ctx(cmd) as ctx: + err = 'fsid does not match ceph.conf' + with pytest.raises(cd.Error, match=err): + retval = cd.command_shell(ctx) + assert retval == 1 + assert ctx.fsid == None + + def test_name(self, cephadm_fs): + cmd = ['shell', '--name', 'foo'] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + + cmd = ['shell', '--name', 'foo.bar'] + with with_cephadm_ctx(cmd) as ctx: + err = r'must pass --fsid' + with pytest.raises(cd.Error, match=err): + retval = cd.command_shell(ctx) + assert retval == 1 + + fsid = '00000000-0000-0000-0000-0000deadbeef' + cmd = ['shell', '--name', 'foo.bar', '--fsid', fsid] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + + def test_config(self, cephadm_fs): + cmd = ['shell'] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert ctx.config == None + + cephadm_fs.create_file(cd.SHELL_DEFAULT_CONF) + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert ctx.config == cd.SHELL_DEFAULT_CONF + + cmd = ['shell', '--config', 'foo'] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert ctx.config == 'foo' + + def test_keyring(self, cephadm_fs): + cmd = ['shell'] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert ctx.keyring == None + + cephadm_fs.create_file(cd.SHELL_DEFAULT_KEYRING) + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert ctx.keyring == cd.SHELL_DEFAULT_KEYRING + + cmd = ['shell', '--keyring', 'foo'] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert ctx.keyring == 'foo' + + +class TestCephVolume(object): + + @staticmethod + def _get_cmd(*args): + return [ + 'ceph-volume', + *args, + '--', 'inventory', '--format', 'json' + ] + + def test_noop(self, cephadm_fs): + cmd = self._get_cmd() + with with_cephadm_ctx(cmd) as ctx: + cd.command_ceph_volume(ctx) + assert ctx.fsid == None + assert ctx.config == None + assert ctx.keyring == None + assert ctx.config_json == None + + def test_fsid(self, cephadm_fs): + fsid = '00000000-0000-0000-0000-0000deadbeef' + + cmd = self._get_cmd('--fsid', fsid) + with with_cephadm_ctx(cmd) as ctx: + cd.command_ceph_volume(ctx) + assert ctx.fsid == fsid + + cmd = self._get_cmd('--fsid', '00000000-0000-0000-0000-0000deadbeez') + with with_cephadm_ctx(cmd) as ctx: + err = 'not an fsid' + with pytest.raises(cd.Error, match=err): + retval = cd.command_shell(ctx) + assert retval == 1 + assert ctx.fsid == None + + s = get_ceph_conf(fsid=fsid) + f = cephadm_fs.create_file('ceph.conf', contents=s) + + cmd = self._get_cmd('--fsid', fsid, '--config', f.path) + with with_cephadm_ctx(cmd) as ctx: + cd.command_ceph_volume(ctx) + assert ctx.fsid == fsid + + cmd = self._get_cmd('--fsid', '10000000-0000-0000-0000-0000deadbeef', '--config', f.path) + with with_cephadm_ctx(cmd) as ctx: + err = 'fsid does not match ceph.conf' + with pytest.raises(cd.Error, match=err): + cd.command_ceph_volume(ctx) + assert ctx.fsid == None + + def test_config(self, cephadm_fs): + cmd = self._get_cmd('--config', 'foo') + with with_cephadm_ctx(cmd) as ctx: + err = r'No such file or directory' + with pytest.raises(cd.Error, match=err): + cd.command_ceph_volume(ctx) + + cephadm_fs.create_file('bar') + cmd = self._get_cmd('--config', 'bar') + with with_cephadm_ctx(cmd) as ctx: + cd.command_ceph_volume(ctx) + assert ctx.config == 'bar' + + def test_keyring(self, cephadm_fs): + cmd = self._get_cmd('--keyring', 'foo') + with with_cephadm_ctx(cmd) as ctx: + err = r'No such file or directory' + with pytest.raises(cd.Error, match=err): + cd.command_ceph_volume(ctx) + + cephadm_fs.create_file('bar') + cmd = self._get_cmd('--keyring', 'bar') + with with_cephadm_ctx(cmd) as ctx: + cd.command_ceph_volume(ctx) + assert ctx.keyring == 'bar' + + +class TestIscsi: + def test_unit_run(self, cephadm_fs): + fsid = '9b9d7609-f4d5-4aba-94c8-effa764d96c9' + config_json = { + 'files': {'iscsi-gateway.cfg': ''} + } + with with_cephadm_ctx(['--image=ceph/ceph'], list_networks={}) as ctx: + import json + ctx.config_json = json.dumps(config_json) + ctx.fsid = fsid + cd.get_parm.return_value = config_json + iscsi = cd.CephIscsi(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9', 'daemon_id', config_json) + c = iscsi.get_tcmu_runner_container() + + cd.make_data_dir(ctx, fsid, 'iscsi', 'daemon_id') + cd.deploy_daemon_units( + ctx, + fsid, + 0, 0, + 'iscsi', + 'daemon_id', + c, + True, True + ) + + with open('/var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/unit.run') as f: + assert f.read() == """set -e +if ! grep -qs /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs /proc/mounts; then mount -t configfs none /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs; fi +# iscsi tcmu-runnter container +! /usr/bin/podman rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi.daemon_id-tcmu 2> /dev/null +! /usr/bin/podman rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu 2> /dev/null +/usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/tcmu-runner --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -e CEPH_USE_RANDOM_NONCE=1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log/rbd-target-api:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph & +# iscsi.daemon_id +! /usr/bin/podman rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi.daemon_id-tcmu 2> /dev/null +! /usr/bin/podman rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu 2> /dev/null +/usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/tcmu-runner --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -e CEPH_USE_RANDOM_NONCE=1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log/rbd-target-api:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph +""" + + def test_get_container(self): + """ + Due to a combination of socket.getfqdn() and podman's behavior to + add the container name into the /etc/hosts file, we cannot use periods + in container names. But we need to be able to detect old existing containers. + Assert this behaviour. I think we can remove this in Ceph R + """ + fsid = '9b9d7609-f4d5-4aba-94c8-effa764d96c9' + with with_cephadm_ctx(['--image=ceph/ceph'], list_networks={}) as ctx: + ctx.fsid = fsid + c = cd.get_container(ctx, fsid, 'iscsi', 'something') + assert c.cname == 'ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-something' + assert c.old_cname == 'ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi.something' + + + diff --git a/ceph/src/cephadm/tests/test_networks.py b/ceph/src/cephadm/tests/test_networks.py new file mode 100644 index 000000000..f7bd44eb8 --- /dev/null +++ b/ceph/src/cephadm/tests/test_networks.py @@ -0,0 +1,201 @@ +import json +from textwrap import dedent +from unittest import mock + +import pytest + +from tests.fixtures import with_cephadm_ctx, cephadm_fs + +with mock.patch('builtins.open', create=True): + from importlib.machinery import SourceFileLoader + cd = SourceFileLoader('cephadm', 'cephadm').load_module() + + +class TestCommandListNetworks: + @pytest.mark.parametrize("test_input, expected", [ + ( + dedent(""" + default via 192.168.178.1 dev enxd89ef3f34260 proto dhcp metric 100 + 10.0.0.0/8 via 10.4.0.1 dev tun0 proto static metric 50 + 10.3.0.0/21 via 10.4.0.1 dev tun0 proto static metric 50 + 10.4.0.1 dev tun0 proto kernel scope link src 10.4.0.2 metric 50 + 137.1.0.0/16 via 10.4.0.1 dev tun0 proto static metric 50 + 138.1.0.0/16 via 10.4.0.1 dev tun0 proto static metric 50 + 139.1.0.0/16 via 10.4.0.1 dev tun0 proto static metric 50 + 140.1.0.0/17 via 10.4.0.1 dev tun0 proto static metric 50 + 141.1.0.0/16 via 10.4.0.1 dev tun0 proto static metric 50 + 169.254.0.0/16 dev docker0 scope link metric 1000 + 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 + 192.168.39.0/24 dev virbr1 proto kernel scope link src 192.168.39.1 linkdown + 192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 linkdown + 192.168.178.0/24 dev enxd89ef3f34260 proto kernel scope link src 192.168.178.28 metric 100 + 192.168.178.1 dev enxd89ef3f34260 proto static scope link metric 100 + 195.135.221.12 via 192.168.178.1 dev enxd89ef3f34260 proto static metric 100 + """), + { + '10.4.0.1': {'tun0': {'10.4.0.2'}}, + '172.17.0.0/16': {'docker0': {'172.17.0.1'}}, + '192.168.39.0/24': {'virbr1': {'192.168.39.1'}}, + '192.168.122.0/24': {'virbr0': {'192.168.122.1'}}, + '192.168.178.0/24': {'enxd89ef3f34260': {'192.168.178.28'}} + } + ), ( + dedent(""" + default via 10.3.64.1 dev eno1 proto static metric 100 + 10.3.64.0/24 dev eno1 proto kernel scope link src 10.3.64.23 metric 100 + 10.3.64.0/24 dev eno1 proto kernel scope link src 10.3.64.27 metric 100 + 10.88.0.0/16 dev cni-podman0 proto kernel scope link src 10.88.0.1 linkdown + 172.21.0.0/20 via 172.21.3.189 dev tun0 + 172.21.1.0/20 via 172.21.3.189 dev tun0 + 172.21.2.1 via 172.21.3.189 dev tun0 + 172.21.3.1 dev tun0 proto kernel scope link src 172.21.3.2 + 172.21.4.0/24 via 172.21.3.1 dev tun0 + 172.21.5.0/24 via 172.21.3.1 dev tun0 + 172.21.6.0/24 via 172.21.3.1 dev tun0 + 172.21.7.0/24 via 172.21.3.1 dev tun0 + 192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 linkdown + 192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 linkdown + 192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 linkdown + 192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 linkdown + """), + { + '10.3.64.0/24': {'eno1': {'10.3.64.23', '10.3.64.27'}}, + '10.88.0.0/16': {'cni-podman0': {'10.88.0.1'}}, + '172.21.3.1': {'tun0': {'172.21.3.2'}}, + '192.168.122.0/24': {'virbr0': {'192.168.122.1'}} + } + ), + ]) + def test_parse_ipv4_route(self, test_input, expected): + assert cd._parse_ipv4_route(test_input) == expected + + @pytest.mark.parametrize("test_routes, test_ips, expected", [ + ( + dedent(""" + ::1 dev lo proto kernel metric 256 pref medium + fe80::/64 dev eno1 proto kernel metric 100 pref medium + fe80::/64 dev br-3d443496454c proto kernel metric 256 linkdown pref medium + fe80::/64 dev tun0 proto kernel metric 256 pref medium + fe80::/64 dev br-4355f5dbb528 proto kernel metric 256 pref medium + fe80::/64 dev docker0 proto kernel metric 256 linkdown pref medium + fe80::/64 dev cni-podman0 proto kernel metric 256 linkdown pref medium + fe80::/64 dev veth88ba1e8 proto kernel metric 256 pref medium + fe80::/64 dev vethb6e5fc7 proto kernel metric 256 pref medium + fe80::/64 dev vethaddb245 proto kernel metric 256 pref medium + fe80::/64 dev vethbd14d6b proto kernel metric 256 pref medium + fe80::/64 dev veth13e8fd2 proto kernel metric 256 pref medium + fe80::/64 dev veth1d3aa9e proto kernel metric 256 pref medium + fe80::/64 dev vethe485ca9 proto kernel metric 256 pref medium + """), + dedent(""" + 1: lo: mtu 65536 state UNKNOWN qlen 1000 + inet6 ::1/128 scope host + valid_lft forever preferred_lft forever + 2: eno1: mtu 1500 state UP qlen 1000 + inet6 fe80::225:90ff:fee5:26e8/64 scope link noprefixroute + valid_lft forever preferred_lft forever + 6: br-3d443496454c: mtu 1500 state DOWN + inet6 fe80::42:23ff:fe9d:ee4/64 scope link + valid_lft forever preferred_lft forever + 7: br-4355f5dbb528: mtu 1500 state UP + inet6 fe80::42:6eff:fe35:41fe/64 scope link + valid_lft forever preferred_lft forever + 8: docker0: mtu 1500 state DOWN + inet6 fe80::42:faff:fee6:40a0/64 scope link + valid_lft forever preferred_lft forever + 11: tun0: mtu 1500 state UNKNOWN qlen 100 + inet6 fe80::98a6:733e:dafd:350/64 scope link stable-privacy + valid_lft forever preferred_lft forever + 28: cni-podman0: mtu 1500 state DOWN qlen 1000 + inet6 fe80::3449:cbff:fe89:b87e/64 scope link + valid_lft forever preferred_lft forever + 31: vethaddb245@if30: mtu 1500 state UP + inet6 fe80::90f7:3eff:feed:a6bb/64 scope link + valid_lft forever preferred_lft forever + 33: veth88ba1e8@if32: mtu 1500 state UP + inet6 fe80::d:f5ff:fe73:8c82/64 scope link + valid_lft forever preferred_lft forever + 35: vethbd14d6b@if34: mtu 1500 state UP + inet6 fe80::b44f:8ff:fe6f:813d/64 scope link + valid_lft forever preferred_lft forever + 37: vethb6e5fc7@if36: mtu 1500 state UP + inet6 fe80::4869:c6ff:feaa:8afe/64 scope link + valid_lft forever preferred_lft forever + 39: veth13e8fd2@if38: mtu 1500 state UP + inet6 fe80::78f4:71ff:fefe:eb40/64 scope link + valid_lft forever preferred_lft forever + 41: veth1d3aa9e@if40: mtu 1500 state UP + inet6 fe80::24bd:88ff:fe28:5b18/64 scope link + valid_lft forever preferred_lft forever + 43: vethe485ca9@if42: mtu 1500 state UP + inet6 fe80::6425:87ff:fe42:b9f0/64 scope link + valid_lft forever preferred_lft forever + """), + { + "fe80::/64": { + "eno1": {"fe80::225:90ff:fee5:26e8"}, + "br-3d443496454c": {"fe80::42:23ff:fe9d:ee4"}, + "tun0": {"fe80::98a6:733e:dafd:350"}, + "br-4355f5dbb528": {"fe80::42:6eff:fe35:41fe"}, + "docker0": {"fe80::42:faff:fee6:40a0"}, + "cni-podman0": {"fe80::3449:cbff:fe89:b87e"}, + "veth88ba1e8": {"fe80::d:f5ff:fe73:8c82"}, + "vethb6e5fc7": {"fe80::4869:c6ff:feaa:8afe"}, + "vethaddb245": {"fe80::90f7:3eff:feed:a6bb"}, + "vethbd14d6b": {"fe80::b44f:8ff:fe6f:813d"}, + "veth13e8fd2": {"fe80::78f4:71ff:fefe:eb40"}, + "veth1d3aa9e": {"fe80::24bd:88ff:fe28:5b18"}, + "vethe485ca9": {"fe80::6425:87ff:fe42:b9f0"}, + } + } + ), + ( + dedent(""" + ::1 dev lo proto kernel metric 256 pref medium + 2001:1458:301:eb::100:1a dev ens20f0 proto kernel metric 100 pref medium + 2001:1458:301:eb::/64 dev ens20f0 proto ra metric 100 pref medium + fd01:1458:304:5e::/64 dev ens20f0 proto ra metric 100 pref medium + fe80::/64 dev ens20f0 proto kernel metric 100 pref medium + default proto ra metric 100 + nexthop via fe80::46ec:ce00:b8a0:d3c8 dev ens20f0 weight 1 + nexthop via fe80::46ec:ce00:b8a2:33c8 dev ens20f0 weight 1 pref medium + """), + dedent(""" + 1: lo: mtu 65536 state UNKNOWN qlen 1000 + inet6 ::1/128 scope host + valid_lft forever preferred_lft forever + 2: ens20f0: mtu 1500 state UP qlen 1000 + inet6 2001:1458:301:eb::100:1a/128 scope global dynamic noprefixroute + valid_lft 590879sec preferred_lft 590879sec + inet6 fe80::2e60:cff:fef8:da41/64 scope link noprefixroute + valid_lft forever preferred_lft forever + inet6 fe80::2e60:cff:fef8:da41/64 scope link noprefixroute + valid_lft forever preferred_lft forever + inet6 fe80::2e60:cff:fef8:da41/64 scope link noprefixroute + valid_lft forever preferred_lft forever + """), + { + '2001:1458:301:eb::/64': { + 'ens20f0': { + '2001:1458:301:eb::100:1a' + }, + }, + 'fe80::/64': { + 'ens20f0': {'fe80::2e60:cff:fef8:da41'}, + }, + 'fd01:1458:304:5e::/64': { + 'ens20f0': set() + }, + } + ), + ]) + def test_parse_ipv6_route(self, test_routes, test_ips, expected): + assert cd._parse_ipv6_route(test_routes, test_ips) == expected + + @mock.patch.object(cd, 'call_throws', return_value=('10.4.0.1 dev tun0 proto kernel scope link src 10.4.0.2 metric 50\n', '', '')) + def test_command_list_networks(self, cephadm_fs, capsys): + with with_cephadm_ctx([]) as ctx: + cd.command_list_networks(ctx) + assert json.loads(capsys.readouterr().out) == { + '10.4.0.1': {'tun0': ['10.4.0.2']} + } \ No newline at end of file diff --git a/ceph/src/client/Client.cc b/ceph/src/client/Client.cc index 7d052bad8..3f344c3fa 100644 --- a/ceph/src/client/Client.cc +++ b/ceph/src/client/Client.cc @@ -1123,7 +1123,7 @@ void Client::update_dentry_lease(Dentry *dn, LeaseStat *dlease, utime_t from, Me /* * update MDS location cache for a single inode */ -void Client::update_dir_dist(Inode *in, DirStat *dst) +void Client::update_dir_dist(Inode *in, DirStat *dst, mds_rank_t from) { // auth ldout(cct, 20) << "got dirfrag map for " << in->ino << " frag " << dst->frag << " to mds " << dst->auth << dendl; @@ -1137,12 +1137,14 @@ void Client::update_dir_dist(Inode *in, DirStat *dst) _fragmap_remove_non_leaves(in); } - // replicated - in->dir_replicated = !dst->dist.empty(); - if (!dst->dist.empty()) - in->frag_repmap[dst->frag].assign(dst->dist.begin(), dst->dist.end()) ; - else - in->frag_repmap.erase(dst->frag); + // replicated, only update from auth mds reply + if (from == dst->auth) { + in->dir_replicated = !dst->dist.empty(); + if (!dst->dist.empty()) + in->frag_repmap[dst->frag].assign(dst->dist.begin(), dst->dist.end()) ; + else + in->frag_repmap.erase(dst->frag); + } } void Client::clear_dir_complete_and_ordered(Inode *diri, bool complete) @@ -1432,7 +1434,8 @@ Inode* Client::insert_trace(MetaRequest *request, MetaSession *session) if (reply->head.is_dentry) { diri = add_update_inode(&dirst, request->sent_stamp, session, request->perms); - update_dir_dist(diri, &dst); // dir stat info is attached to .. + mds_rank_t from_mds = mds_rank_t(reply->get_source().num()); + update_dir_dist(diri, &dst, from_mds); // dir stat info is attached to .. if (in) { Dir *dir = diri->open_dir(); @@ -6313,9 +6316,29 @@ void Client::_close_sessions() } } +void Client::flush_mdlog_sync(Inode *in) +{ + if (in->unsafe_ops.empty()) { + return; + } + + std::set anchor; + for (auto &&p : in->unsafe_ops) { + anchor.emplace(p->mds); + } + if (in->auth_cap) { + anchor.emplace(in->auth_cap->session->mds_num); + } + + for (auto &rank : anchor) { + auto session = &mds_sessions.at(rank); + flush_mdlog(session); + } +} + void Client::flush_mdlog_sync() { - if (mds_requests.empty()) + if (mds_requests.empty()) return; for (auto &p : mds_sessions) { flush_mdlog(&p.second); @@ -10565,7 +10588,7 @@ int Client::_fsync(Inode *in, bool syncdataonly) } else ldout(cct, 10) << "no metadata needs to commit" << dendl; if (!syncdataonly && !in->unsafe_ops.empty()) { - flush_mdlog_sync(); + flush_mdlog_sync(in); MetaRequest *req = in->unsafe_ops.back(); ldout(cct, 15) << "waiting on unsafe requests, last tid " << req->get_tid() << dendl; diff --git a/ceph/src/client/Client.h b/ceph/src/client/Client.h index 88e8f4429..2884cee55 100644 --- a/ceph/src/client/Client.h +++ b/ceph/src/client/Client.h @@ -776,7 +776,7 @@ public: void unlock_fh_pos(Fh *f); // metadata cache - void update_dir_dist(Inode *in, DirStat *st); + void update_dir_dist(Inode *in, DirStat *st, mds_rank_t from); void clear_dir_complete_and_ordered(Inode *diri, bool complete); void insert_readdir_results(MetaRequest *request, MetaSession *session, Inode *diri); @@ -797,6 +797,7 @@ public: vinodeno_t map_faked_ino(ino_t ino); //notify the mds to flush the mdlog + void flush_mdlog_sync(Inode *in); void flush_mdlog_sync(); void flush_mdlog(MetaSession *session); diff --git a/ceph/src/cls/cmpomap/client.h b/ceph/src/cls/cmpomap/client.h index c55fbc410..013d85cc7 100644 --- a/ceph/src/cls/cmpomap/client.h +++ b/ceph/src/cls/cmpomap/client.h @@ -26,7 +26,8 @@ static constexpr uint32_t max_keys = 1000; /// process each of the omap value comparisons according to the same rules as /// cmpxattr(), and return -ECANCELED if a comparison is unsuccessful. for /// comparisons with Mode::U64, failure to decode an input value is reported -/// as -EINVAL, while failure to decode a stored value is reported as -EIO +/// as -EINVAL, an empty stored value is compared as 0, and failure to decode +/// a stored value is reported as -EIO [[nodiscard]] int cmp_vals(librados::ObjectReadOperation& op, Mode mode, Op comparison, ComparisonMap values, std::optional default_value); @@ -34,9 +35,9 @@ static constexpr uint32_t max_keys = 1000; /// process each of the omap value comparisons according to the same rules as /// cmpxattr(). any key/value pairs that compare successfully are overwritten /// with the corresponding input value. for comparisons with Mode::U64, failure -/// to decode an input value is reported as -EINVAL. decode failure of a stored -/// value is treated as an unsuccessful comparison and is not reported as an -/// error +/// to decode an input value is reported as -EINVAL. an empty stored value is +/// compared as 0, while decode failure of a stored value is treated as an +/// unsuccessful comparison and is not reported as an error [[nodiscard]] int cmp_set_vals(librados::ObjectWriteOperation& writeop, Mode mode, Op comparison, ComparisonMap values, std::optional default_value); @@ -44,8 +45,9 @@ static constexpr uint32_t max_keys = 1000; /// process each of the omap value comparisons according to the same rules as /// cmpxattr(). any key/value pairs that compare successfully are removed. for /// comparisons with Mode::U64, failure to decode an input value is reported as -/// -EINVAL. decode failure of a stored value is treated as an unsuccessful -/// comparison and is not reported as an error +/// -EINVAL. an empty stored value is compared as 0, while decode failure of a +/// stored value is treated as an unsuccessful comparison and is not reported +/// as an error [[nodiscard]] int cmp_rm_keys(librados::ObjectWriteOperation& writeop, Mode mode, Op comparison, ComparisonMap values); diff --git a/ceph/src/cls/cmpomap/server.cc b/ceph/src/cls/cmpomap/server.cc index 691832bfe..86e16d940 100644 --- a/ceph/src/cls/cmpomap/server.cc +++ b/ceph/src/cls/cmpomap/server.cc @@ -37,17 +37,20 @@ static int compare_values(Op op, const T& lhs, const T& rhs) static int compare_values_u64(Op op, uint64_t lhs, const bufferlist& value) { - try { - // decode existing value as rhs - uint64_t rhs; - auto p = value.cbegin(); - using ceph::decode; - decode(rhs, p); - return compare_values(op, lhs, rhs); - } catch (const buffer::error&) { - // failures to decode existing values are reported as EIO - return -EIO; + // empty values compare as 0 for backward compat + uint64_t rhs = 0; + if (value.length()) { + try { + // decode existing value as rhs + auto p = value.cbegin(); + using ceph::decode; + decode(rhs, p); + } catch (const buffer::error&) { + // failures to decode existing values are reported as EIO + return -EIO; + } } + return compare_values(op, lhs, rhs); } static int compare_value(Mode mode, Op op, const bufferlist& input, diff --git a/ceph/src/cls/rgw/cls_rgw.cc b/ceph/src/cls/rgw/cls_rgw.cc index bc455a04b..cc2d786a6 100644 --- a/ceph/src/cls/rgw/cls_rgw.cc +++ b/ceph/src/cls/rgw/cls_rgw.cc @@ -501,6 +501,7 @@ int rgw_bucket_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out) bool has_delimiter = !op.delimiter.empty(); if (has_delimiter && + start_after_key > op.filter_prefix && boost::algorithm::ends_with(start_after_key, op.delimiter)) { // advance past all subdirectory entries if we start after a // subdirectory @@ -1195,6 +1196,7 @@ class BIVerObjEntry { public: BIVerObjEntry(cls_method_context_t& _hctx, const cls_rgw_obj_key& _key) : hctx(_hctx), key(_key), initialized(false) { + // empty } int init(bool check_delete_marker = true) { @@ -1532,11 +1534,20 @@ static int rgw_bucket_link_olh(cls_method_context_t hctx, bufferlist *in, buffer return -EINVAL; } - BIVerObjEntry obj(hctx, op.key); - BIOLHEntry olh(hctx, op.key); - /* read instance entry */ + BIVerObjEntry obj(hctx, op.key); int ret = obj.init(op.delete_marker); + + /* NOTE: When a delete is issued, a key instance is always provided, + * either the one for which the delete is requested or a new random + * one when no instance is specified. So we need to see which of + * these two cases we're dealing with. The variable `existed` will + * be true if the instance was specified and false if it was + * randomly generated. It might have been cleaner if the instance + * were empty and randomly generated here and returned in the reply, + * as that would better allow a typo in the instance id. This code + * should be audited and possibly cleaned up. */ + bool existed = (ret == 0); if (ret == -ENOENT && op.delete_marker) { ret = 0; @@ -1545,6 +1556,28 @@ static int rgw_bucket_link_olh(cls_method_context_t hctx, bufferlist *in, buffer return ret; } + BIOLHEntry olh(hctx, op.key); + bool olh_read_attempt = false; + bool olh_found = false; + if (!existed && op.delete_marker) { + /* read olh */ + ret = olh.init(&olh_found); + if (ret < 0) { + return ret; + } + olh_read_attempt = true; + + // if we're deleting (i.e., adding a delete marker, and the OLH + // indicates it already refers to a delete marker, error out) + if (olh_found && olh.get_entry().delete_marker) { + CLS_LOG(10, + "%s: delete marker received for \"%s\" although OLH" + " already refers to a delete marker\n", + __func__, escape_str(op.key.to_string()).c_str()); + return -ENOENT; + } + } + if (existed && !real_clock::is_zero(op.unmod_since)) { timespec mtime = ceph::real_clock::to_timespec(obj.mtime()); timespec unmod = ceph::real_clock::to_timespec(op.unmod_since); @@ -1597,11 +1630,14 @@ static int rgw_bucket_link_olh(cls_method_context_t hctx, bufferlist *in, buffer } /* read olh */ - bool olh_found; - ret = olh.init(&olh_found); - if (ret < 0) { - return ret; + if (!olh_read_attempt) { // only read if we didn't attempt earlier + ret = olh.init(&olh_found); + if (ret < 0) { + return ret; + } + olh_read_attempt = true; } + const uint64_t prev_epoch = olh.get_epoch(); if (!olh.start_modify(op.olh_epoch)) { diff --git a/ceph/src/cls/rgw/cls_rgw_types.h b/ceph/src/cls/rgw/cls_rgw_types.h index 434feceee..f70c108c8 100644 --- a/ceph/src/cls/rgw/cls_rgw_types.h +++ b/ceph/src/cls/rgw/cls_rgw_types.h @@ -1,13 +1,16 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab -#ifndef CEPH_CLS_RGW_TYPES_H -#define CEPH_CLS_RGW_TYPES_H +#pragma once #include #include "common/ceph_time.h" #include "common/Formatter.h" +#undef FMT_HEADER_ONLY +#define FMT_HEADER_ONLY 1 +#include + #include "rgw/rgw_basic_types.h" #define CEPH_RGW_REMOVE 'r' @@ -340,6 +343,14 @@ struct cls_rgw_obj_key { cls_rgw_obj_key(const std::string &_name) : name(_name) {} cls_rgw_obj_key(const std::string& n, const std::string& i) : name(n), instance(i) {} + std::string to_string() const { + return fmt::format("{}({})", name, instance); + } + + bool empty() const { + return name.empty(); + } + void set(const std::string& _name) { name = _name; } @@ -348,6 +359,7 @@ struct cls_rgw_obj_key { return (name.compare(k.name) == 0) && (instance.compare(k.instance) == 0); } + bool operator<(const cls_rgw_obj_key& k) const { int r = name.compare(k.name); if (r == 0) { @@ -355,12 +367,16 @@ struct cls_rgw_obj_key { } return (r < 0); } + bool operator<=(const cls_rgw_obj_key& k) const { return !(k < *this); } - bool empty() const { - return name.empty(); + + std::ostream& operator<<(std::ostream& out) const { + out << to_string(); + return out; } + void encode(ceph::buffer::list &bl) const { ENCODE_START(1, 1, bl); encode(name, bl); @@ -1283,5 +1299,3 @@ struct cls_rgw_reshard_entry void get_key(std::string *key) const; }; WRITE_CLASS_ENCODER(cls_rgw_reshard_entry) - -#endif diff --git a/ceph/src/common/CMakeLists.txt b/ceph/src/common/CMakeLists.txt index 6f29dfef3..e1f1dca47 100644 --- a/ceph/src/common/CMakeLists.txt +++ b/ceph/src/common/CMakeLists.txt @@ -177,6 +177,7 @@ target_compile_definitions(common-common-objs PRIVATE "CEPH_LIBDIR=\"${CMAKE_INSTALL_FULL_LIBDIR}\"" "CEPH_PKGLIBDIR=\"${CEPH_INSTALL_FULL_PKGLIBDIR}\"" "CEPH_DATADIR=\"${CEPH_INSTALL_DATADIR}\"") +compile_with_fmt(common-common-objs) set(common_mountcephfs_srcs armor.c @@ -213,7 +214,7 @@ elseif(HAVE_ARMV8_CRC) crc32c_aarch64.c) endif(HAVE_INTEL) -add_library(crc32 ${crc32_srcs}) +add_library(crc32 STATIC ${crc32_srcs}) if(HAVE_ARMV8_CRC) set_target_properties(crc32 PROPERTIES COMPILE_FLAGS "${CMAKE_C_FLAGS} ${ARMV8_CRC_COMPILE_FLAGS}") diff --git a/ceph/src/common/Formatter.cc b/ceph/src/common/Formatter.cc index b599e48b3..362deffb5 100644 --- a/ceph/src/common/Formatter.cc +++ b/ceph/src/common/Formatter.cc @@ -19,6 +19,7 @@ #include "include/buffer.h" #include +#include #include #include diff --git a/ceph/src/common/buffer.cc b/ceph/src/common/buffer.cc index 406ca24a4..31154c96f 100644 --- a/ceph/src/common/buffer.cc +++ b/ceph/src/common/buffer.cc @@ -1254,8 +1254,12 @@ static ceph::spinlock debug_lock; buffer::create_aligned(unaligned._len, align_memory))); had_to_rebuild = true; } - _buffers.insert_after(p_prev, *ptr_node::create(unaligned._buffers.front()).release()); - _num += 1; + if (unaligned.get_num_buffers()) { + _buffers.insert_after(p_prev, *ptr_node::create(unaligned._buffers.front()).release()); + _num += 1; + } else { + // a bufferlist containing only 0-length bptrs is rebuilt as empty + } ++p_prev; } return had_to_rebuild; diff --git a/ceph/src/common/ipaddr.cc b/ceph/src/common/ipaddr.cc index 5eddc9b16..8c5da54b9 100644 --- a/ceph/src/common/ipaddr.cc +++ b/ceph/src/common/ipaddr.cc @@ -5,7 +5,6 @@ #include #include #include -#include #if defined(__FreeBSD__) #include #include @@ -33,54 +32,23 @@ void netmask_ipv4(const struct in_addr *addr, out->s_addr = addr->s_addr & mask; } - -static bool match_numa_node(const string& if_name, int numa_node) +bool matches_ipv4_in_subnet(const struct ifaddrs& addrs, + const struct sockaddr_in* net, + unsigned int prefix_len) { -#if defined(WITH_SEASTAR) || defined(_WIN32) - return true; -#else - int if_node = -1; - int r = get_iface_numa_node(if_name, &if_node); - if (r < 0) { + if (addrs.ifa_addr == nullptr) return false; - } - return if_node == numa_node; -#endif -} - -const struct ifaddrs *find_ipv4_in_subnet(const struct ifaddrs *addrs, - const struct sockaddr_in *net, - unsigned int prefix_len, - int numa_node) { - struct in_addr want, temp; + if (addrs.ifa_addr->sa_family != net->sin_family) + return false; + struct in_addr want; netmask_ipv4(&net->sin_addr, prefix_len, &want); - for (; addrs != NULL; addrs = addrs->ifa_next) { - - if (addrs->ifa_addr == NULL) - continue; - - if (strcmp(addrs->ifa_name, "lo") == 0 || boost::starts_with(addrs->ifa_name, "lo:")) - continue; - - if (numa_node >= 0 && !match_numa_node(addrs->ifa_name, numa_node)) - continue; - - if (addrs->ifa_addr->sa_family != net->sin_family) - continue; - - struct in_addr *cur = &((struct sockaddr_in*)addrs->ifa_addr)->sin_addr; - netmask_ipv4(cur, prefix_len, &temp); - - if (temp.s_addr == want.s_addr) { - return addrs; - } - } - - return NULL; + struct in_addr *cur = &((struct sockaddr_in*)addrs.ifa_addr)->sin_addr; + struct in_addr temp; + netmask_ipv4(cur, prefix_len, &temp); + return temp.s_addr == want.s_addr; } - void netmask_ipv6(const struct in6_addr *addr, unsigned int prefix_len, struct in6_addr *out) { @@ -94,59 +62,25 @@ void netmask_ipv6(const struct in6_addr *addr, memset(out->s6_addr+prefix_len/8+1, 0, 16-prefix_len/8-1); } +bool matches_ipv6_in_subnet(const struct ifaddrs& addrs, + const struct sockaddr_in6* net, + unsigned int prefix_len) +{ + if (addrs.ifa_addr == nullptr) + return false; -const struct ifaddrs *find_ipv6_in_subnet(const struct ifaddrs *addrs, - const struct sockaddr_in6 *net, - unsigned int prefix_len, - int numa_node) { - struct in6_addr want, temp; - + if (addrs.ifa_addr->sa_family != net->sin6_family) + return false; + struct in6_addr want; netmask_ipv6(&net->sin6_addr, prefix_len, &want); - for (; addrs != NULL; addrs = addrs->ifa_next) { - - if (addrs->ifa_addr == NULL) - continue; - - if (strcmp(addrs->ifa_name, "lo") == 0 || boost::starts_with(addrs->ifa_name, "lo:")) - continue; - - if (numa_node >= 0 && !match_numa_node(addrs->ifa_name, numa_node)) - continue; - - if (addrs->ifa_addr->sa_family != net->sin6_family) - continue; - - struct in6_addr *cur = &((struct sockaddr_in6*)addrs->ifa_addr)->sin6_addr; - if (IN6_IS_ADDR_LINKLOCAL(cur)) - continue; - netmask_ipv6(cur, prefix_len, &temp); - - if (IN6_ARE_ADDR_EQUAL(&temp, &want)) - return addrs; - } - - return NULL; -} - - -const struct ifaddrs *find_ip_in_subnet(const struct ifaddrs *addrs, - const struct sockaddr *net, - unsigned int prefix_len, - int numa_node) { - switch (net->sa_family) { - case AF_INET: - return find_ipv4_in_subnet(addrs, (struct sockaddr_in*)net, prefix_len, - numa_node); - - case AF_INET6: - return find_ipv6_in_subnet(addrs, (struct sockaddr_in6*)net, prefix_len, - numa_node); - } - - return NULL; + struct in6_addr temp; + struct in6_addr *cur = &((struct sockaddr_in6*)addrs.ifa_addr)->sin6_addr; + if (IN6_IS_ADDR_LINKLOCAL(cur)) + return false; + netmask_ipv6(cur, prefix_len, &temp); + return IN6_ARE_ADDR_EQUAL(&temp, &want); } - bool parse_network(const char *s, struct sockaddr_storage *network, unsigned int *prefix_len) { char *slash = strchr((char*)s, '/'); if (!slash) { diff --git a/ceph/src/common/legacy_config_opts.h b/ceph/src/common/legacy_config_opts.h index 8e2438bf0..f2610f33a 100644 --- a/ceph/src/common/legacy_config_opts.h +++ b/ceph/src/common/legacy_config_opts.h @@ -1445,7 +1445,6 @@ OPTION(rgw_data_log_num_shards, OPT_INT) // number of objects to keep data chang OPTION(rgw_data_log_obj_prefix, OPT_STR) // OPTION(rgw_bucket_quota_ttl, OPT_INT) // time for cached bucket stats to be cached within rgw instance -OPTION(rgw_bucket_quota_soft_threshold, OPT_DOUBLE) // threshold from which we don't rely on cached info for quota decisions OPTION(rgw_bucket_quota_cache_size, OPT_INT) // number of entries in bucket quota cache OPTION(rgw_bucket_default_quota_max_objects, OPT_INT) // number of objects allowed OPTION(rgw_bucket_default_quota_max_size, OPT_LONGLONG) // Max size of object in bytes @@ -1509,6 +1508,10 @@ OPTION(rgw_crypt_vault_addr, OPT_STR) // Vault server base address OPTION(rgw_crypt_vault_prefix, OPT_STR) // Optional URL prefix to Vault secret path OPTION(rgw_crypt_vault_secret_engine, OPT_STR) // kv, transit or other supported secret engines OPTION(rgw_crypt_vault_namespace, OPT_STR) // Vault Namespace (only availabe in Vault Enterprise Version) +OPTION(rgw_crypt_vault_verify_ssl, OPT_BOOL) // should we try to verify vault's ssl +OPTION(rgw_crypt_vault_ssl_cacert, OPT_STR) // optional ca certificate for accessing vault +OPTION(rgw_crypt_vault_ssl_clientcert, OPT_STR) // client certificate for accessing vault +OPTION(rgw_crypt_vault_ssl_clientkey, OPT_STR) // private key for client certificate OPTION(rgw_crypt_kmip_addr, OPT_STR) // kmip server address OPTION(rgw_crypt_kmip_ca_path, OPT_STR) // ca for kmip servers diff --git a/ceph/src/common/options.cc b/ceph/src/common/options.cc index 96eabd2d9..bdec3edaf 100644 --- a/ceph/src/common/options.cc +++ b/ceph/src/common/options.cc @@ -1153,6 +1153,11 @@ std::vector