From b3b6e05ebb17ef6ac03c6f6f46d242185719a08e Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Mon, 12 Jul 2021 14:38:07 +0200 Subject: [PATCH] import ceph pacific 16.2.5 Signed-off-by: Thomas Lamprecht --- ceph/.github/labeler.yml | 3 + ceph/CMakeLists.txt | 3 +- ceph/PendingReleaseNotes | 37 + ceph/ceph.spec | 7 +- ceph/ceph.spec.in | 1 + ceph/changelog.upstream | 10 +- ceph/debian/ceph-mgr-modules-core.install | 1 + ceph/debian/control | 2 - ceph/debian/libsqlite3-mod-ceph.symbols | 37 + ceph/doc/cephadm/adoption.rst | 4 +- ceph/doc/cephadm/compatibility.rst | 6 + ceph/doc/cephadm/host-management.rst | 70 +- ceph/doc/cephadm/install.rst | 34 +- ceph/doc/cephadm/monitoring.rst | 137 +- ceph/doc/cephadm/nfs.rst | 94 +- ceph/doc/cephadm/operations.rst | 79 +- ceph/doc/cephadm/osd.rst | 200 +- ceph/doc/cephadm/rgw.rst | 21 +- ceph/doc/cephadm/service-management.rst | 246 +- ceph/doc/cephadm/upgrade.rst | 113 +- ceph/doc/cephfs/administration.rst | 41 +- ceph/doc/cephfs/cephfs-mirroring.rst | 385 ++ ceph/doc/cephfs/fs-nfs-exports.rst | 53 +- ceph/doc/cephfs/index.rst | 2 +- ceph/doc/cephfs/multimds.rst | 25 +- ceph/doc/cephfs/snap-schedule.rst | 6 +- ceph/doc/cephfs/upgrading.rst | 6 + ceph/doc/dev/cephadm/developing-cephadm.rst | 144 + ceph/doc/dev/cephfs-mirroring.rst | 61 +- ceph/doc/dev/developer_guide/dash-devel.rst | 42 +- ceph/doc/install/get-packages.rst | 9 +- ceph/doc/man/8/cephfs-top.rst | 36 + ceph/doc/mgr/mds_autoscaler.rst | 36 +- ceph/doc/rados/configuration/index.rst | 1 + .../rados/configuration/mclock-config-ref.rst | 368 ++ .../rados/configuration/osd-config-ref.rst | 16 +- ceph/doc/rbd/rbd-live-migration.rst | 2 +- .../rbd/rbd-persistent-write-back-cache.rst | 22 +- ceph/make-dist | 16 +- .../grafana/dashboards/host-details.json | 38 +- .../grafana/dashboards/hosts-overview.json | 4 +- .../grafana/dashboards/pool-overview.json | 1586 +++-- .../cephfs/clusters/1-node-1-mds-1-osd.yaml | 8 + ceph/qa/distros/all/rhel_8.4.yaml | 6 + ceph/qa/distros/all/rhel_8.yaml | 2 +- .../podman/centos_8.2_kubic_stable.yaml | 2 + .../distros/podman/rhel_8.3_kubic_stable.yaml | 2 + .../podman/ubuntu_18.04_kubic_stable.yaml | 3 +- .../podman/ubuntu_20.04_kubic_stable.yaml | 3 +- .../podman/ubuntu_20.04_kubic_testing.yaml | 3 +- .../supported-all-distro/ubuntu_latest.yaml | 2 +- .../ubuntu_latest.yaml | 2 +- .../bugs/client_trim_caps/centos_latest.yaml | 1 + .../{rados/cephadm/dashboard => fs/full}/% | 0 .../thrash-old-clients => fs/full}/.qa | 0 ceph/qa/suites/fs/full/begin.yaml | 1 + .../full/clusters}/.qa | 0 .../fs/full/clusters/1-node-1-mds-1-osd.yaml | 1 + ceph/qa/suites/fs/full/conf | 1 + ceph/qa/suites/fs/full/distro | 1 + ceph/qa/suites/fs/full/mount/fuse.yaml | 2 + .../1-install => fs/full/objectstore}/.qa | 0 .../fs/full/objectstore/bluestore-bitmap.yaml | 1 + ceph/qa/suites/fs/full/overrides.yaml | 19 + .../clusters => fs/full/overrides}/+ | 0 .../backoff => fs/full/overrides}/.qa | 0 .../suites/fs/full/overrides/frag_enable.yaml | 1 + .../fs/full/overrides/no_client_pidfile.yaml | 1 + .../fs/full/overrides/whitelist_health.yaml | 1 + .../whitelist_wrongly_marked_down.yaml | 1 + .../clusters => fs/full/tasks}/.qa | 0 .../qa/suites/fs/full/tasks/mgr-osd-full.yaml | 21 + .../fs/mirror/overrides/whitelist_health.yaml | 15 +- .../suites/fs/shell/tasks/cephfs-shell.yaml | 2 +- .../old_client/centos_latest.yaml | 1 + .../old_client/tasks/0-nautilus.yaml | 5 + .../upgraded_client/centos_latest.yaml | 1 + .../upgraded_client/tasks/0-nautilus.yaml | 5 + .../import-legacy/tasks/0-nautilus.yaml | 5 + .../mirror/overrides/whitelist_health.yaml | 15 +- ceph/qa/suites/fs/verify/distro~HEAD | 1 - .../% => fs/workload/wsync/$} | 0 ceph/qa/suites/fs/workload/wsync/no.yaml | 3 + ceph/qa/suites/fs/workload/wsync/yes.yaml | 3 + ceph/qa/suites/{rados/cephadm => orch}/.qa | 0 .../cephadm/dashboard => orch/cephadm}/.qa | 0 .../cephadm/dashboard}/% | 0 .../0-distro => orch/cephadm/dashboard}/.qa | 0 .../cephadm/dashboard/0-distro}/.qa | 0 .../0-distro/centos_8.2_kubic_stable.yaml | 0 .../cephadm/dashboard/task/test_e2e.yaml | 0 .../smoke => orch/cephadm/orchestrator_cli}/% | 0 .../cephadm/orchestrator_cli}/.qa | 0 .../cephadm/orchestrator_cli/0-random-distro$ | 0 .../cephadm/orchestrator_cli/2-node-mgr.yaml | 0 .../orchestrator_cli/orchestrator_cli.yaml | 0 .../cephadm/smoke-roleless}/% | 0 .../smoke => orch/cephadm/smoke-roleless}/.qa | 0 .../cephadm/smoke-roleless/0-distro | 0 .../cephadm/smoke-roleless/1-start.yaml | 0 .../smoke-roleless/2-services/basic.yaml | 0 .../2-services/client-keyring.yaml | 40 + .../smoke-roleless/2-services/iscsi.yaml | 0 .../smoke-roleless/2-services/mirror.yaml | 0 .../2-services/nfs-ingress.yaml | 68 + .../2-services/nfs-ingress2.yaml | 50 + .../smoke-roleless/2-services/nfs.yaml | 13 + .../smoke-roleless/2-services/nfs2.yaml | 12 + .../2-services/rgw-ingress.yaml | 0 .../smoke-roleless/2-services/rgw.yaml | 0 .../cephadm/smoke-roleless/3-final.yaml | 0 .../cephadm/smoke-singlehost}/% | 0 .../orch/cephadm/smoke-singlehost/0-distro$ | 1 + .../cephadm/smoke-singlehost/1-start.yaml | 27 + .../smoke-singlehost/2-services/basic.yaml} | 0 .../smoke-singlehost/2-services/rgw.yaml | 12 + .../cephadm/smoke-singlehost/3-final.yaml | 8 + .../cephadm/upgrade => orch/cephadm/smoke}/% | 0 .../smoke/distro => orch/cephadm/smoke}/.qa | 0 .../thrash => orch/cephadm/smoke/distro}/.qa | 0 .../smoke/distro/centos_8.2_kubic_stable.yaml | 0 .../smoke/distro/rhel_8.3_kubic_stable.yaml | 0 .../cephadm/smoke/distro/ubuntu_18.04.yaml | 0 .../cephadm/smoke/distro/ubuntu_20.04.yaml | 0 .../cephadm/smoke/fixed-2.yaml | 0 .../cephadm/smoke/mon_election | 0 .../{rados => orch}/cephadm/smoke/start.yaml | 0 .../cephadm/thrash-old-clients}/% | 0 .../cephadm/thrash-old-clients}/.qa | 0 .../0-size-min-size-overrides}/.qa | 0 .../2-size-2-min-size.yaml | 0 .../3-size-2-min-size.yaml | 0 .../cephadm/thrash-old-clients/1-install}/.qa | 0 .../1-install/luminous-v1only.yaml | 0 .../1-install/luminous.yaml | 0 .../1-install/mimic-v1only.yaml | 0 .../thrash-old-clients/1-install/mimic.yaml | 0 .../1-install/nautilus-v1only.yaml | 0 .../1-install/nautilus-v2only.yaml | 0 .../1-install/nautilus.yaml | 0 .../thrash-old-clients/1-install/octopus.yaml | 0 .../cephadm/thrash-old-clients/backoff}/.qa | 0 .../thrash-old-clients/backoff/normal.yaml} | 0 .../thrash-old-clients/backoff/peering.yaml | 0 .../backoff/peering_and_degraded.yaml | 0 .../cephadm/thrash-old-clients/ceph.yaml | 0 .../cephadm/thrash-old-clients/clusters/+} | 0 .../cephadm/thrash-old-clients/clusters}/.qa | 0 .../clusters/openstack.yaml | 0 .../clusters/three-plus-one.yaml | 0 .../thrash-old-clients/d-balancer}/.qa | 0 .../d-balancer/crush-compat.yaml | 0 .../thrash-old-clients/d-balancer/on.yaml} | 0 .../cephadm/thrash-old-clients/distro$}/.qa | 0 .../distro$/ubuntu_18.04.yaml | 0 .../cephadm/thrash-old-clients/mon_election | 0 .../thrash-old-clients/msgr-failures}/.qa | 0 .../msgr-failures/fastclose.yaml | 0 .../thrash-old-clients/msgr-failures/few.yaml | 0 .../msgr-failures/osd-delay.yaml | 0 .../cephadm/thrash-old-clients/rados.yaml | 0 .../cephadm/thrash-old-clients/thrashers}/.qa | 0 .../thrash-old-clients/thrashers/careful.yaml | 0 .../thrash-old-clients/thrashers/default.yaml | 0 .../thrash-old-clients/thrashers/mapgap.yaml | 0 .../thrashers/morepggrow.yaml | 0 .../thrash-old-clients/thrashers/none.yaml} | 0 .../thrash-old-clients/thrashers/pggrow.yaml | 0 .../thrash-old-clients/thrashosds-health.yaml | 0 .../cephadm/thrash-old-clients/workloads/.qa | 1 + .../workloads/cache-snaps.yaml | 0 .../workloads/radosbench.yaml | 0 .../thrash-old-clients/workloads/rbd_cls.yaml | 0 .../workloads/snaps-few-objects.yaml | 0 .../workloads/test_rbd_api.yaml | 0 ceph/qa/suites/orch/cephadm/thrash/% | 0 .../upgrade => orch/cephadm/thrash}/.qa | 0 .../{rados => orch}/cephadm/thrash/0-distro | 0 .../cephadm/thrash/1-start.yaml | 0 .../cephadm/thrash/2-thrash.yaml | 0 .../qa/suites/orch/cephadm/thrash/3-tasks/.qa | 1 + .../thrash/3-tasks/rados_api_tests.yaml | 1 + .../cephadm/thrash/3-tasks/radosbench.yaml | 1 + .../cephadm/thrash/3-tasks/small-objects.yaml | 1 + .../thrash/3-tasks/snaps-few-objects.yaml | 1 + .../cephadm/thrash/fixed-2.yaml | 0 .../{rados => orch}/cephadm/thrash/msgr | 0 .../{rados => orch}/cephadm/thrash/root.yaml | 0 ceph/qa/suites/orch/cephadm/upgrade/% | 0 .../workunits => orch/cephadm/upgrade}/.qa | 0 .../1-start-centos_8.3-octopus.yaml | 1 + .../1-start-ubuntu_20.04-15.2.9.yaml | 1 + .../1-start-distro/1-start-ubuntu_20.04.yaml | 1 + .../cephadm/upgrade/2-repo_digest/defaut.yaml | 0 .../upgrade/2-repo_digest/repo_digest.yaml | 0 .../cephadm/upgrade/3-start-upgrade.yaml | 0 .../cephadm/upgrade/4-wait.yaml | 0 .../cephadm/upgrade/mon_election | 0 ceph/qa/suites/orch/cephadm/with-work/% | 0 ceph/qa/suites/orch/cephadm/with-work/.qa | 1 + .../cephadm/with-work/0-distro | 0 .../cephadm/with-work/fixed-2.yaml | 0 .../qa/suites/orch/cephadm/with-work/mode/.qa | 1 + .../cephadm/with-work/mode/packaged.yaml | 0 .../cephadm/with-work/mode/root.yaml | 0 .../cephadm/with-work/mon_election | 0 .../{rados => orch}/cephadm/with-work/msgr | 0 .../cephadm/with-work/start.yaml | 0 .../suites/orch/cephadm/with-work/tasks/.qa | 1 + .../with-work/tasks/rados_api_tests.yaml | 1 + .../cephadm/with-work/tasks/rados_python.yaml | 1 + ceph/qa/suites/orch/cephadm/workunits/% | 0 .../0-distro => orch/cephadm/workunits}/.qa | 0 .../orch/cephadm/workunits/0-distro/.qa | 1 + .../0-distro/centos_8.2_kubic_stable.yaml | 0 .../cephadm/workunits/mon_election | 0 .../cephadm/workunits/task/test_adoption.yaml | 0 .../cephadm/workunits/task/test_cephadm.yaml | 0 .../workunits/task/test_cephadm_repos.yaml | 0 .../cephadm/workunits/task/test_orch_cli.yaml | 0 ceph/qa/suites/orch/rook/.qa | 1 + ceph/qa/suites/orch/rook/smoke/% | 0 ceph/qa/suites/orch/rook/smoke/.qa | 1 + ceph/qa/suites/orch/rook/smoke/0-distro/.qa | 1 + .../rook/smoke/0-distro/ubuntu_18.04.yaml | 2 + .../rook/smoke/0-distro/ubuntu_20.04.yaml | 2 + ceph/qa/suites/orch/rook/smoke/0-kubeadm.yaml | 2 + ceph/qa/suites/orch/rook/smoke/1-rook.yaml | 7 + .../orch/rook/smoke/2-workload/none.yaml | 0 .../rook/smoke/2-workload/radosbench.yaml | 5 + ceph/qa/suites/orch/rook/smoke/3-final.yaml | 8 + .../orch/rook/smoke/cluster/1-node.yaml | 3 + .../orch/rook/smoke/cluster/3-node.yaml | 7 + ceph/qa/suites/orch/rook/smoke/k8s/1.21.yaml | 3 + .../qa/suites/orch/rook/smoke/net/calico.yaml | 3 + .../qa/suites/orch/rook/smoke/rook/1.6.2.yaml | 4 + .../suites/orch/rook/smoke/rook/master.yaml | 3 + ceph/qa/suites/rados/cephadm | 1 + .../thrash/3-tasks/rados_api_tests.yaml | 1 - .../cephadm/thrash/3-tasks/radosbench.yaml | 1 - .../cephadm/thrash/3-tasks/small-objects.yaml | 1 - .../thrash/3-tasks/snaps-few-objects.yaml | 1 - .../with-work/tasks/rados_api_tests.yaml | 1 - .../cephadm/with-work/tasks/rados_python.yaml | 1 - ceph/qa/suites/rados/rook | 1 + ceph/qa/suites/rados/standalone/mon_election | 1 - .../suites/rados/thrash/3-scrub-overrides/$ | 0 .../suites/rados/thrash/3-scrub-overrides/.qa | 1 + .../thrash/3-scrub-overrides/default.yaml | 0 .../max-simultaneous-scrubs-2.yaml | 5 + .../max-simultaneous-scrubs-3.yaml | 5 + .../msgr-failures/osd-dispatch-delay.yaml | 7 + .../upgrade/octopus-x/parallel/1-tasks.yaml | 1 + .../octopus-x/stress-split/1-start.yaml | 1 + .../pacific-p2p/pacific-p2p-parallel/% | 0 .../point-to-point-upgrade.yaml | 177 + .../supported-all-distro/ubuntu_latest.yaml | 2 + .../pacific-p2p/pacific-p2p-stress-split/% | 0 .../pacific-p2p-stress-split/0-cluster/+ | 0 .../0-cluster/openstack.yaml | 6 + .../0-cluster/start.yaml | 33 + .../1-ceph-install/pacific..yaml | 21 + .../1.1.short_pg_log.yaml | 6 + .../2-partial-upgrade/firsthalf.yaml | 13 + .../3-thrash/default.yaml | 27 + .../pacific-p2p-stress-split/4-workload/+ | 0 .../4-workload/fsx.yaml | 8 + .../4-workload/radosbench.yaml | 52 + .../4-workload/rbd-cls.yaml | 10 + .../4-workload/rbd-import-export.yaml | 12 + .../4-workload/rbd_api.yaml | 10 + .../4-workload/readwrite.yaml | 16 + .../4-workload/snaps-few-objects.yaml | 18 + .../5-finish-upgrade.yaml | 8 + .../6-final-workload/+ | 0 .../6-final-workload/rbd-python.yaml | 10 + .../6-final-workload/snaps-many-objects.yaml | 16 + .../objectstore/bluestore-bitmap.yaml | 43 + .../objectstore/bluestore-comp.yaml | 23 + .../objectstore/bluestore-stupid.yaml | 43 + .../objectstore/filestore-xfs.yaml | 15 + .../supported-all-distro/ubuntu_latest.yaml | 2 + .../thrashosds-health.yaml | 15 + ceph/qa/tasks/ceph_fuse.py | 4 +- ceph/qa/tasks/ceph_manager.py | 33 +- ceph/qa/tasks/cephadm.conf | 2 + ceph/qa/tasks/cephadm.py | 87 +- ceph/qa/tasks/cephadm_cases/test_cli.py | 2 +- ceph/qa/tasks/cephfs/test_data_scan.py | 4 +- ceph/qa/tasks/cephfs/test_exports.py | 17 +- ceph/qa/tasks/cephfs/test_forward_scrub.py | 6 +- ceph/qa/tasks/cephfs/test_mirroring.py | 390 +- ceph/qa/tasks/cephfs/test_multimds_misc.py | 8 +- ceph/qa/tasks/cephfs/test_nfs.py | 82 +- ceph/qa/tasks/cephfs/test_recovery_pool.py | 2 +- ceph/qa/tasks/cephfs/test_scrub.py | 4 +- ceph/qa/tasks/cephfs/test_snap_schedules.py | 36 +- ceph/qa/tasks/cephfs/test_volume_client.py | 13 +- ceph/qa/tasks/daemonwatchdog.py | 11 +- ceph/qa/tasks/fwd_scrub.py | 6 +- ceph/qa/tasks/kclient.py | 4 +- ceph/qa/tasks/kubeadm.py | 536 ++ .../qa/tasks/mgr/dashboard/test_crush_rule.py | 3 +- ceph/qa/tasks/mgr/dashboard/test_host.py | 2 +- ceph/qa/tasks/mgr/dashboard/test_rgw.py | 10 + ceph/qa/tasks/mgr/test_dashboard.py | 6 + ceph/qa/tasks/mgr/test_progress.py | 2 +- ceph/qa/tasks/rook-ceph.conf | 41 + ceph/qa/tasks/rook.py | 638 ++ ceph/qa/tasks/userdata_setup.yaml | 2 + ceph/qa/tasks/vip.py | 33 + ceph/qa/tasks/workunit.py | 1 + ceph/qa/workunits/cephadm/test_cephadm.sh | 7 +- .../workunits/cephadm/test_dashboard_e2e.sh | 17 +- ceph/qa/workunits/fs/full/subvolume_rm.sh | 62 + ceph/qa/workunits/rbd/cli_generic.sh | 216 +- ceph/qa/workunits/rbd/qemu-iotests.sh | 9 +- ceph/qa/workunits/rgw/test_rgw_orphan_list.sh | 20 +- ceph/src/.git_version | 4 +- ceph/src/CMakeLists.txt | 7 + ceph/src/SimpleRADOSStriper.h | 2 +- .../ceph_volume/devices/lvm/batch.py | 3 +- ceph/src/ceph_mon.cc | 15 +- ceph/src/cephadm/cephadm | 533 +- ceph/src/cephadm/tests/fixtures.py | 79 +- ceph/src/cephadm/tests/test_cephadm.py | 287 +- ceph/src/cephadm/tox.ini | 29 +- ceph/src/client/Client.cc | 585 +- ceph/src/client/Client.h | 39 +- ceph/src/client/Inode.cc | 46 +- ceph/src/client/Inode.h | 135 +- ceph/src/cls/fifo/cls_fifo.cc | 17 +- ceph/src/cls/rgw/cls_rgw.cc | 82 +- ceph/src/common/buffer.cc | 8 +- ceph/src/common/config.cc | 7 +- ceph/src/common/config.h | 5 + ceph/src/common/config_proxy.h | 3 + ceph/src/common/dout.h | 5 + ceph/src/common/options.cc | 36 +- ceph/src/crush/crush.h | 2 +- ceph/src/include/ceph_fs.h | 12 +- ceph/src/include/cephfs/libcephfs.h | 138 + ceph/src/include/cephfs/metrics/Types.h | 3 +- ceph/src/include/fs_types.h | 4 + ceph/src/include/libcephsqlite.h | 8 +- ceph/src/include/mempool.h | 2 +- ceph/src/include/win32/fs_compat.h | 1 + ceph/src/libcephfs.cc | 89 + ceph/src/librbd/api/Trash.cc | 64 +- ceph/src/librbd/cache/pwl/ImageCacheState.cc | 25 +- .../mirror/snapshot/CreatePrimaryRequest.cc | 12 + ceph/src/logrotate.conf | 2 +- ceph/src/mds/CDir.cc | 2 +- ceph/src/mds/CInode.cc | 6 +- ceph/src/mds/CInode.h | 2 +- ceph/src/mds/DamageTable.cc | 7 +- ceph/src/mds/Locker.cc | 12 + ceph/src/mds/MDBalancer.cc | 2 +- ceph/src/mds/MDCache.cc | 156 +- ceph/src/mds/MDCache.h | 3 +- ceph/src/mds/MDLog.cc | 36 +- ceph/src/mds/MDSAuthCaps.cc | 19 +- ceph/src/mds/MDSAuthCaps.h | 12 +- ceph/src/mds/MDSContext.cc | 7 +- ceph/src/mds/MDSDaemon.cc | 1 + ceph/src/mds/MDSRank.cc | 12 +- ceph/src/mds/MDSRank.h | 11 +- ceph/src/mds/MDSTable.cc | 4 +- ceph/src/mds/Mutation.h | 1 + ceph/src/mds/OpenFileTable.cc | 10 +- ceph/src/mds/Server.cc | 65 +- ceph/src/mds/Server.h | 3 +- ceph/src/mds/SessionMap.cc | 13 +- ceph/src/mds/SnapRealm.cc | 2 +- ceph/src/mds/mdstypes.h | 13 +- ceph/src/messages/MMonJoin.h | 22 +- ceph/src/messages/MMonProbe.h | 10 +- ceph/src/mgr/ActivePyModules.cc | 74 +- ceph/src/mgr/BaseMgrModule.cc | 9 + ceph/src/mgr/BaseMgrStandbyModule.cc | 40 + ceph/src/mgr/DaemonServer.cc | 6 +- ceph/src/mgr/Gil.cc | 35 + ceph/src/mgr/Gil.h | 42 + ceph/src/mgr/MgrStandby.cc | 17 +- ceph/src/mgr/PyModuleRegistry.h | 4 + ceph/src/mgr/PyOSDMap.cc | 15 + ceph/src/mgr/StandbyPyModules.h | 3 + ceph/src/mon/Elector.cc | 1 + ceph/src/mon/Elector.h | 2 + ceph/src/mon/FSCommands.cc | 4 + ceph/src/mon/MDSMonitor.cc | 18 +- ceph/src/mon/MonClient.cc | 22 +- ceph/src/mon/MonClient.h | 14 +- ceph/src/mon/Monitor.cc | 126 +- ceph/src/mon/Monitor.h | 25 +- ceph/src/mon/MonmapMonitor.cc | 33 +- ceph/src/mon/OSDMonitor.cc | 164 +- ceph/src/mon/OSDMonitor.h | 45 +- ceph/src/msg/async/ProtocolV2.cc | 2 +- ceph/src/os/bluestore/AvlAllocator.cc | 43 +- ceph/src/os/bluestore/BlueFS.cc | 50 +- ceph/src/os/bluestore/BlueFS.h | 36 +- ceph/src/os/bluestore/BlueRocksEnv.cc | 56 +- ceph/src/os/bluestore/BlueRocksEnv.h | 8 - ceph/src/os/bluestore/BlueStore.cc | 35 +- ceph/src/os/bluestore/BlueStore.h | 34 +- ceph/src/os/bluestore/HybridAllocator.cc | 2 + ceph/src/os/bluestore/StupidAllocator.cc | 4 + ceph/src/os/bluestore/bluefs_types.h | 8 +- .../src/os/filestore/BtrfsFileStoreBackend.cc | 12 +- ceph/src/os/filestore/FileStore.cc | 12 +- ceph/src/os/filestore/LFNIndex.cc | 28 +- ceph/src/osd/OSD.cc | 138 +- ceph/src/osd/OSD.h | 12 +- ceph/src/osd/OSDMap.cc | 1 - ceph/src/osd/PG.cc | 17 +- ceph/src/osd/PrimaryLogPG.cc | 2 +- ceph/src/osd/osd_types.cc | 4 +- ceph/src/osd/scheduler/mClockScheduler.cc | 40 +- ceph/src/osd/scheduler/mClockScheduler.h | 3 - ceph/src/pybind/ceph_argparse.py | 42 +- ceph/src/pybind/ceph_volume_client.py | 23 +- ceph/src/pybind/cephfs/c_cephfs.pxd | 2 + ceph/src/pybind/cephfs/cephfs.pyx | 29 + ceph/src/pybind/cephfs/mock_cephfs.pxi | 4 + ceph/src/pybind/mgr/ceph_module.pyi | 3 + ceph/src/pybind/mgr/cephadm/autotune.py | 54 + ceph/src/pybind/mgr/cephadm/inventory.py | 275 +- ceph/src/pybind/mgr/cephadm/module.py | 300 +- ceph/src/pybind/mgr/cephadm/remotes.py | 19 + ceph/src/pybind/mgr/cephadm/schedule.py | 159 +- ceph/src/pybind/mgr/cephadm/serve.py | 326 +- .../mgr/cephadm/services/cephadmservice.py | 61 +- .../pybind/mgr/cephadm/services/ingress.py | 61 +- ceph/src/pybind/mgr/cephadm/services/iscsi.py | 57 +- .../pybind/mgr/cephadm/services/monitoring.py | 34 +- ceph/src/pybind/mgr/cephadm/services/nfs.py | 183 +- ceph/src/pybind/mgr/cephadm/services/osd.py | 68 +- .../services/grafana/ceph-dashboard.yml.j2 | 2 +- .../templates/services/grafana/grafana.ini.j2 | 1 + .../templates/services/ingress/haproxy.cfg.j2 | 26 +- .../templates/services/nfs/ganesha.conf.j2 | 4 + .../services/prometheus/prometheus.yml.j2 | 3 +- ceph/src/pybind/mgr/cephadm/tests/fixtures.py | 17 +- .../pybind/mgr/cephadm/tests/test_autotune.py | 69 + .../pybind/mgr/cephadm/tests/test_cephadm.py | 256 +- .../mgr/cephadm/tests/test_scheduling.py | 219 +- .../pybind/mgr/cephadm/tests/test_services.py | 6 + .../pybind/mgr/cephadm/tests/test_upgrade.py | 62 +- ceph/src/pybind/mgr/cephadm/upgrade.py | 57 +- ceph/src/pybind/mgr/cephadm/utils.py | 11 + .../dashboard/ci/cephadm/bootstrap-cluster.sh | 22 + .../mgr/dashboard/ci/cephadm/ceph_cluster.yml | 40 + .../ci/cephadm/run-cephadm-e2e-tests.sh | 81 + .../mgr/dashboard/ci/check_grafana_uids.py | 50 +- .../mgr/dashboard/controllers/__init__.py | 65 +- .../mgr/dashboard/controllers/crush_rule.py | 6 +- .../pybind/mgr/dashboard/controllers/docs.py | 35 +- .../pybind/mgr/dashboard/controllers/host.py | 45 +- .../mgr/dashboard/controllers/prometheus.py | 2 +- .../pybind/mgr/dashboard/controllers/rgw.py | 10 +- .../mgr/dashboard/controllers/service.py | 4 +- .../mgr/dashboard/frontend/cypress.json | 1 + .../fixtures/orchestrator/services.json | 523 ++ .../integration/cluster/crush-map.e2e-spec.ts | 4 +- .../cypress/integration/cluster/hosts.po.ts | 83 +- .../integration/cluster/services.po.ts | 84 + .../filesystems/filesystems.e2e-spec.ts | 4 +- .../01-hosts-force-maintenance.e2e-spec.ts | 39 + .../orchestrator/01-hosts.e2e-spec.ts | 17 +- .../orchestrator/05-services.e2e-spec.ts | 29 + .../workflow/01-hosts.e2e-spec.ts | 57 + .../integration/rgw/buckets.e2e-spec.ts | 18 + .../cypress/integration/rgw/buckets.po.ts | 41 +- .../cypress/integration/ui/navigation.po.ts | 2 +- ...4310d2d34.js => 1.1c3aa698fdc6e2e06036.js} | 2 +- .../frontend/dist/en-US/3rdpartylicenses.txt | 2 +- .../dist/en-US/5.7ef5282604f6c3e77fd4.js | 1 + .../dist/en-US/5.f1a521f29ba388aefe88.js | 1 - .../dist/en-US/6.5aa9ef51cf028c8fa150.js | 1 - .../dist/en-US/6.93b626d621206efed808.js | 1 + .../dashboard/frontend/dist/en-US/index.html | 8 +- .../dist/en-US/main.686c64fd8301e15fd70a.js | 3 + .../dist/en-US/main.abdc8ba4d5af20b6adbf.js | 3 - .../dist/en-US/main.e73fdb683302952f5ca0.css | 19 - ...77c.js => runtime.95ade215c318c70e8872.js} | 2 +- .../en-US/scripts.38cee5fbd56812617717.js | 1 - .../en-US/scripts.6bda3fa7e09a87cd4228.js | 1 + .../en-US/styles.0b9eb5355e15caa2c87d.css | 19 - .../en-US/styles.7918cb8dc788b3eedc95.css | 19 + .../mgr/dashboard/frontend/package-lock.json | 5168 +++++++++-------- .../mgr/dashboard/frontend/package.json | 18 +- .../frontend/src/app/app-routing.module.ts | 4 +- .../dashboard/frontend/src/app/app.module.ts | 5 + .../rbd-details/rbd-details.component.html | 6 +- .../app/ceph/block/rbd-form/rbd-form.model.ts | 3 + .../block/rbd-list/rbd-list.component.html | 22 + .../block/rbd-list/rbd-list.component.scss | 5 + .../block/rbd-list/rbd-list.component.spec.ts | 43 +- .../ceph/block/rbd-list/rbd-list.component.ts | 23 +- .../crushmap/crushmap.component.spec.ts | 171 +- .../cluster/crushmap/crushmap.component.ts | 21 +- .../host-details/host-details.component.html | 3 +- .../hosts/host-form/host-form.component.html | 47 + .../host-form/host-form.component.spec.ts | 40 +- .../hosts/host-form/host-form.component.ts | 22 +- .../cluster/hosts/hosts.component.spec.ts | 47 + .../app/ceph/cluster/hosts/hosts.component.ts | 3 +- .../cluster/services/placement.pipe.spec.ts | 4 +- .../ceph/cluster/services/placement.pipe.ts | 2 +- .../service-daemon-list.component.html | 78 +- .../service-daemon-list.component.scss | 13 + .../service-daemon-list.component.spec.ts | 33 + .../service-daemon-list.component.ts | 67 +- .../service-form/service-form.component.html | 141 +- .../service-form.component.spec.ts | 68 + .../service-form/service-form.component.ts | 58 +- .../services/services.component.spec.ts | 4 - .../cluster/services/services.component.ts | 28 +- .../health-pie/health-pie.component.spec.ts | 3 +- .../health-pie/health-pie.component.ts | 25 +- .../dashboard/health/health.component.spec.ts | 4 +- .../ceph/dashboard/health/health.component.ts | 15 +- .../ceph/dashboard/osd-summary.pipe.spec.ts | 43 +- .../app/ceph/dashboard/osd-summary.pipe.ts | 2 +- .../ceph/nfs/nfs-form/nfs-form.component.html | 3 +- .../nfs/nfs-form/nfs-form.component.spec.ts | 26 +- .../ceph/nfs/nfs-form/nfs-form.component.ts | 5 +- .../rgw-bucket-details.component.html | 5 - .../rgw-bucket-details.component.ts | 1 + .../rgw-bucket-form.component.html | 42 +- .../rgw-bucket-form.component.spec.ts | 173 +- .../rgw-bucket-form.component.ts | 150 +- .../rgw-bucket-list.component.spec.ts | 12 +- .../rgw-bucket-list.component.ts | 20 +- .../rgw-user-form.component.spec.ts | 4 +- .../rgw-user-list/rgw-user-list.component.ts | 2 +- .../workbench-layout.component.spec.ts | 3 +- .../dashboard-help.component.html | 15 +- .../dashboard-help.component.ts | 8 +- .../navigation/navigation.component.html | 10 +- .../navigation/navigation.component.ts | 6 +- .../src/app/shared/api/auth.service.ts | 9 +- .../src/app/shared/api/host.service.ts | 24 +- .../app/shared/api/rgw-bucket.service.spec.ts | 16 +- .../src/app/shared/api/rgw-bucket.service.ts | 39 +- .../src/app/shared/classes/cd-helper.class.ts | 2 +- .../src/app/shared/classes/css-helper.ts | 5 + .../grafana/grafana.component.spec.ts | 16 +- .../components/grafana/grafana.component.ts | 2 + .../src/app/shared/enum/color.enum.ts | 8 - .../src/app/shared/enum/health-color.enum.ts | 5 + .../src/app/shared/forms/cd-validators.ts | 19 +- .../shared/pipes/health-color.pipe.spec.ts | 33 +- .../src/app/shared/pipes/health-color.pipe.ts | 9 +- .../change-password-guard.service.spec.ts | 18 +- .../services/change-password-guard.service.ts | 16 +- .../shared/services/favicon.service.spec.ts | 3 +- .../app/shared/services/favicon.service.ts | 10 +- .../frontend/src/app/shared/shared.module.ts | 3 +- .../mgr/dashboard/frontend/src/index.html | 4 - .../mgr/dashboard/frontend/src/setupJest.ts | 6 + .../mgr/dashboard/frontend/src/styles.scss | 21 - .../dashboard/frontend/src/styles.scss.d.ts | 25 - .../styles/defaults/_bootstrap-defaults.scss | 19 +- .../frontend/src/styles/vendor/_index.scss | 20 + ceph/src/pybind/mgr/dashboard/module.py | 5 +- ceph/src/pybind/mgr/dashboard/openapi.yaml | 20 +- .../mgr/dashboard/run-backend-api-tests.sh | 4 +- .../mgr/dashboard/services/access_control.py | 25 + .../mgr/dashboard/services/orchestrator.py | 4 +- ceph/src/pybind/mgr/dashboard/services/rbd.py | 38 +- .../mgr/dashboard/services/rgw_client.py | 24 +- .../dashboard/tests/test_access_control.py | 109 + .../pybind/mgr/dashboard/tests/test_docs.py | 21 +- .../pybind/mgr/dashboard/tests/test_host.py | 36 +- .../mgr/dashboard/tests/test_rbd_service.py | 98 +- .../mgr/dashboard/tests/test_rgw_client.py | 43 + .../mgr/dashboard/tests/test_versioning.py | 10 + ceph/src/pybind/mgr/devicehealth/module.py | 2 +- ceph/src/pybind/mgr/mgr_module.py | 22 + ceph/src/pybind/mgr/mgr_util.py | 1 + .../pybind/mgr/mirroring/fs/dir_map/policy.py | 40 +- .../mgr/mirroring/fs/snapshot_mirror.py | 122 +- ceph/src/pybind/mgr/mirroring/fs/utils.py | 1 + ceph/src/pybind/mgr/nfs/__init__.py | 1 + ceph/src/pybind/mgr/nfs/cluster.py | 250 + ceph/src/pybind/mgr/nfs/exception.py | 31 + ceph/src/pybind/mgr/nfs/export.py | 488 ++ ceph/src/pybind/mgr/nfs/export_utils.py | 330 ++ ceph/src/pybind/mgr/nfs/module.py | 104 + ceph/src/pybind/mgr/nfs/utils.py | 36 + .../src/pybind/mgr/orchestrator/_interface.py | 75 + ceph/src/pybind/mgr/orchestrator/module.py | 45 +- ceph/src/pybind/mgr/progress/module.py | 60 +- ceph/src/pybind/mgr/prometheus/module.py | 52 +- ceph/src/pybind/mgr/restful/module.py | 2 +- ceph/src/pybind/mgr/snap_schedule/module.py | 7 +- ceph/src/pybind/mgr/tests/__init__.py | 1 + ceph/src/pybind/mgr/volumes/fs/nfs.py | 1010 ---- ceph/src/pybind/mgr/volumes/module.py | 133 - .../deployment/drive_selection/matchers.py | 4 +- .../python-common/ceph/deployment/hostspec.py | 69 +- .../ceph/deployment/service_spec.py | 86 +- .../ceph/tests/test_disk_selector.py | 7 + .../python-common/ceph/tests/test_hostspec.py | 40 + ceph/src/rgw/cls_fifo_legacy.cc | 558 +- ceph/src/rgw/cls_fifo_legacy.h | 67 +- ceph/src/rgw/librgw.cc | 41 +- ceph/src/rgw/rgw_acl.cc | 19 +- ceph/src/rgw/rgw_acl.h | 4 +- ceph/src/rgw/rgw_acl_s3.cc | 44 +- ceph/src/rgw/rgw_acl_s3.h | 4 +- ceph/src/rgw/rgw_acl_swift.cc | 41 +- ceph/src/rgw/rgw_acl_swift.h | 11 +- ceph/src/rgw/rgw_admin.cc | 662 +-- ceph/src/rgw/rgw_auth.cc | 31 +- ceph/src/rgw/rgw_auth.h | 3 +- ceph/src/rgw/rgw_auth_filters.h | 6 +- ceph/src/rgw/rgw_auth_s3.cc | 78 +- ceph/src/rgw/rgw_auth_s3.h | 20 +- ceph/src/rgw/rgw_bucket.cc | 627 +- ceph/src/rgw/rgw_bucket.h | 119 +- ceph/src/rgw/rgw_bucket_sync.cc | 6 +- ceph/src/rgw/rgw_bucket_sync.h | 2 +- ceph/src/rgw/rgw_cache.cc | 49 +- ceph/src/rgw/rgw_cache.h | 15 +- ceph/src/rgw/rgw_common.cc | 10 +- ceph/src/rgw/rgw_common.h | 8 +- ceph/src/rgw/rgw_coroutine.cc | 28 +- ceph/src/rgw/rgw_coroutine.h | 18 +- ceph/src/rgw/rgw_cors_s3.cc | 24 +- ceph/src/rgw/rgw_cors_s3.h | 10 +- ceph/src/rgw/rgw_cr_rados.cc | 164 +- ceph/src/rgw/rgw_cr_rados.h | 210 +- ceph/src/rgw/rgw_cr_rest.cc | 10 +- ceph/src/rgw/rgw_cr_rest.h | 26 +- ceph/src/rgw/rgw_cr_tools.cc | 51 +- ceph/src/rgw/rgw_crypt.cc | 70 +- ceph/src/rgw/rgw_crypt.h | 2 +- ceph/src/rgw/rgw_data_sync.cc | 257 +- ceph/src/rgw/rgw_data_sync.h | 57 +- ceph/src/rgw/rgw_datalog.cc | 194 +- ceph/src/rgw/rgw_datalog.h | 46 +- ceph/src/rgw/rgw_dencoder.cc | 159 - ceph/src/rgw/rgw_etag_verifier.cc | 15 +- ceph/src/rgw/rgw_etag_verifier.h | 3 +- ceph/src/rgw/rgw_file.cc | 10 +- ceph/src/rgw/rgw_file.h | 12 +- ceph/src/rgw/rgw_frontend.h | 19 +- ceph/src/rgw/rgw_gc.cc | 10 +- ceph/src/rgw/rgw_json_enc.cc | 7 +- ceph/src/rgw/rgw_lc.cc | 130 +- ceph/src/rgw/rgw_lc.h | 3 +- ceph/src/rgw/rgw_lib.h | 8 +- ceph/src/rgw/rgw_lib_frontend.h | 2 +- ceph/src/rgw/rgw_loadgen.cc | 5 +- ceph/src/rgw/rgw_loadgen.h | 2 +- ceph/src/rgw/rgw_loadgen_process.cc | 4 +- ceph/src/rgw/rgw_log.cc | 24 +- ceph/src/rgw/rgw_log_backing.cc | 251 +- ceph/src/rgw/rgw_log_backing.h | 113 +- ceph/src/rgw/rgw_lua.cc | 22 +- ceph/src/rgw/rgw_lua.h | 12 +- ceph/src/rgw/rgw_lua_request.cc | 6 +- ceph/src/rgw/rgw_main.cc | 9 +- ceph/src/rgw/rgw_mdlog.h | 17 +- ceph/src/rgw/rgw_metadata.cc | 107 +- ceph/src/rgw/rgw_metadata.h | 57 +- ceph/src/rgw/rgw_multi.cc | 62 +- ceph/src/rgw/rgw_multi.h | 14 +- ceph/src/rgw/rgw_notify.cc | 157 +- ceph/src/rgw/rgw_notify.h | 15 +- ceph/src/rgw/rgw_obj_manifest.cc | 215 +- ceph/src/rgw/rgw_obj_manifest.h | 88 +- ceph/src/rgw/rgw_object_expirer.cc | 3 +- ceph/src/rgw/rgw_object_expirer_core.cc | 98 +- ceph/src/rgw/rgw_object_expirer_core.h | 34 +- ceph/src/rgw/rgw_oidc_provider.cc | 50 +- ceph/src/rgw/rgw_oidc_provider.h | 12 +- ceph/src/rgw/rgw_op.cc | 556 +- ceph/src/rgw/rgw_op.h | 23 +- ceph/src/rgw/rgw_orphan.cc | 317 +- ceph/src/rgw/rgw_orphan.h | 40 +- ceph/src/rgw/rgw_os_lib.cc | 2 +- ceph/src/rgw/rgw_otp.cc | 25 +- ceph/src/rgw/rgw_otp.h | 7 +- ceph/src/rgw/rgw_period_history.cc | 12 +- ceph/src/rgw/rgw_period_history.h | 5 +- ceph/src/rgw/rgw_period_puller.cc | 30 +- ceph/src/rgw/rgw_period_puller.h | 2 +- ceph/src/rgw/rgw_period_pusher.cc | 33 +- ceph/src/rgw/rgw_period_pusher.h | 2 +- ceph/src/rgw/rgw_process.cc | 6 +- ceph/src/rgw/rgw_process.h | 13 +- ceph/src/rgw/rgw_pubsub.cc | 148 +- ceph/src/rgw/rgw_pubsub.h | 42 +- ceph/src/rgw/rgw_pubsub_push.cc | 10 +- ceph/src/rgw/rgw_putobj_processor.cc | 28 +- ceph/src/rgw/rgw_quota.cc | 108 +- ceph/src/rgw/rgw_quota.h | 2 +- ceph/src/rgw/rgw_rados.cc | 1478 ++--- ceph/src/rgw/rgw_rados.h | 339 +- ceph/src/rgw/rgw_realm_reloader.cc | 26 +- ceph/src/rgw/rgw_realm_watcher.cc | 22 +- ceph/src/rgw/rgw_realm_watcher.h | 4 +- ceph/src/rgw/rgw_reshard.cc | 208 +- ceph/src/rgw/rgw_reshard.h | 51 +- ceph/src/rgw/rgw_rest.cc | 54 +- ceph/src/rgw/rgw_rest_bucket.cc | 26 +- ceph/src/rgw/rgw_rest_client.cc | 74 +- ceph/src/rgw/rgw_rest_client.h | 26 +- ceph/src/rgw/rgw_rest_config.cc | 4 +- ceph/src/rgw/rgw_rest_conn.cc | 48 +- ceph/src/rgw/rgw_rest_conn.h | 43 +- ceph/src/rgw/rgw_rest_iam.cc | 6 +- ceph/src/rgw/rgw_rest_log.cc | 165 +- ceph/src/rgw/rgw_rest_metadata.cc | 26 +- ceph/src/rgw/rgw_rest_oidc_provider.cc | 14 +- ceph/src/rgw/rgw_rest_pubsub.cc | 98 +- ceph/src/rgw/rgw_rest_pubsub_common.cc | 64 +- ceph/src/rgw/rgw_rest_realm.cc | 70 +- ceph/src/rgw/rgw_rest_role.cc | 38 +- ceph/src/rgw/rgw_rest_s3.cc | 96 +- ceph/src/rgw/rgw_rest_s3.h | 2 +- ceph/src/rgw/rgw_rest_s3website.h | 2 +- ceph/src/rgw/rgw_rest_sts.cc | 40 +- ceph/src/rgw/rgw_rest_sts.h | 2 +- ceph/src/rgw/rgw_rest_swift.cc | 34 +- ceph/src/rgw/rgw_rest_usage.cc | 4 +- ceph/src/rgw/rgw_rest_user.cc | 74 +- ceph/src/rgw/rgw_rest_user_policy.cc | 42 +- ceph/src/rgw/rgw_role.cc | 109 +- ceph/src/rgw/rgw_role.h | 25 +- ceph/src/rgw/rgw_sal.h | 95 +- ceph/src/rgw/rgw_sal_rados.cc | 278 +- ceph/src/rgw/rgw_sal_rados.h | 117 +- ceph/src/rgw/rgw_service.cc | 102 +- ceph/src/rgw/rgw_service.h | 20 +- ceph/src/rgw/rgw_sts.cc | 24 +- ceph/src/rgw/rgw_sts.h | 6 +- ceph/src/rgw/rgw_swift_auth.cc | 14 +- ceph/src/rgw/rgw_sync.cc | 246 +- ceph/src/rgw/rgw_sync.h | 46 +- ceph/src/rgw/rgw_sync_checkpoint.cc | 9 +- ceph/src/rgw/rgw_sync_error_repo.cc | 8 +- ceph/src/rgw/rgw_sync_module.cc | 8 +- ceph/src/rgw/rgw_sync_module.h | 2 +- ceph/src/rgw/rgw_sync_module_aws.cc | 60 +- ceph/src/rgw/rgw_sync_module_es.cc | 30 +- ceph/src/rgw/rgw_sync_module_es_rest.cc | 16 +- ceph/src/rgw/rgw_sync_module_log.cc | 4 +- ceph/src/rgw/rgw_sync_module_pubsub.cc | 143 +- ceph/src/rgw/rgw_sync_module_pubsub_rest.cc | 28 +- ceph/src/rgw/rgw_sync_trace.cc | 5 +- ceph/src/rgw/rgw_tools.cc | 42 +- ceph/src/rgw/rgw_tools.h | 23 +- ceph/src/rgw/rgw_torrent.cc | 16 +- ceph/src/rgw/rgw_trim_bilog.cc | 143 +- ceph/src/rgw/rgw_trim_bilog.h | 7 +- ceph/src/rgw/rgw_trim_datalog.cc | 53 +- ceph/src/rgw/rgw_trim_datalog.h | 6 +- ceph/src/rgw/rgw_trim_mdlog.cc | 87 +- ceph/src/rgw/rgw_usage.cc | 12 +- ceph/src/rgw/rgw_usage.h | 7 +- ceph/src/rgw/rgw_user.cc | 437 +- ceph/src/rgw/rgw_user.h | 164 +- ceph/src/rgw/rgw_worker.h | 13 +- ceph/src/rgw/rgw_zone.cc | 505 +- ceph/src/rgw/rgw_zone.h | 120 +- ceph/src/rgw/services/svc_bi.h | 10 +- ceph/src/rgw/services/svc_bi_rados.cc | 90 +- ceph/src/rgw/services/svc_bi_rados.h | 35 +- ceph/src/rgw/services/svc_bilog_rados.cc | 23 +- ceph/src/rgw/services/svc_bilog_rados.h | 13 +- ceph/src/rgw/services/svc_bucket.h | 23 +- ceph/src/rgw/services/svc_bucket_sobj.cc | 105 +- ceph/src/rgw/services/svc_bucket_sobj.h | 31 +- ceph/src/rgw/services/svc_bucket_sync.h | 12 +- ceph/src/rgw/services/svc_bucket_sync_sobj.cc | 137 +- ceph/src/rgw/services/svc_bucket_sync_sobj.h | 22 +- ceph/src/rgw/services/svc_cls.cc | 121 +- ceph/src/rgw/services/svc_cls.h | 49 +- ceph/src/rgw/services/svc_config_key_rados.cc | 2 +- ceph/src/rgw/services/svc_config_key_rados.h | 2 +- ceph/src/rgw/services/svc_finisher.cc | 2 +- ceph/src/rgw/services/svc_finisher.h | 2 +- ceph/src/rgw/services/svc_mdlog.cc | 129 +- ceph/src/rgw/services/svc_mdlog.h | 23 +- ceph/src/rgw/services/svc_meta_be.cc | 47 +- ceph/src/rgw/services/svc_meta_be.h | 56 +- ceph/src/rgw/services/svc_meta_be_otp.cc | 10 +- ceph/src/rgw/services/svc_meta_be_otp.h | 6 +- ceph/src/rgw/services/svc_meta_be_sobj.cc | 33 +- ceph/src/rgw/services/svc_meta_be_sobj.h | 17 +- ceph/src/rgw/services/svc_notify.cc | 69 +- ceph/src/rgw/services/svc_notify.h | 15 +- ceph/src/rgw/services/svc_otp.cc | 32 +- ceph/src/rgw/services/svc_otp.h | 20 +- ceph/src/rgw/services/svc_rados.cc | 32 +- ceph/src/rgw/services/svc_rados.h | 16 +- ceph/src/rgw/services/svc_sync_modules.cc | 4 +- ceph/src/rgw/services/svc_sync_modules.h | 2 +- ceph/src/rgw/services/svc_sys_obj.cc | 67 +- ceph/src/rgw/services/svc_sys_obj.h | 40 +- ceph/src/rgw/services/svc_sys_obj_cache.cc | 114 +- ceph/src/rgw/services/svc_sys_obj_cache.h | 38 +- ceph/src/rgw/services/svc_sys_obj_core.cc | 175 +- ceph/src/rgw/services/svc_sys_obj_core.h | 52 +- .../src/rgw/services/svc_sys_obj_core_types.h | 3 +- ceph/src/rgw/services/svc_user.h | 40 +- ceph/src/rgw/services/svc_user_rados.cc | 232 +- ceph/src/rgw/services/svc_user_rados.h | 71 +- ceph/src/rgw/services/svc_zone.cc | 374 +- ceph/src/rgw/services/svc_zone.h | 38 +- ceph/src/rgw/services/svc_zone_utils.cc | 2 +- ceph/src/rgw/services/svc_zone_utils.h | 2 +- ceph/src/rocksdb/CMakeLists.txt | 5 + .../rocksdb/build_tools/build_detect_platform | 14 + ceph/src/rocksdb/util/crc32c.cc | 10 +- ceph/src/rocksdb/util/crc32c_arm64.cc | 124 +- ceph/src/rocksdb/util/crc32c_arm64.h | 1 + ceph/src/test/cls_rgw/test_cls_rgw.cc | 11 +- ceph/src/test/libcephfs/test.cc | 917 ++- ceph/src/test/librados/service.cc | 4 +- .../librbd/migration/test_mock_HttpClient.cc | 2 +- .../test_mock_CreatePrimaryRequest.cc | 87 +- ceph/src/test/librbd/test_mirroring.cc | 8 +- ceph/src/test/librbd/test_notify.py | 21 +- ceph/src/test/objectstore/Allocator_test.cc | 58 + .../test/objectstore/allocator_replay_test.cc | 15 +- ceph/src/test/objectstore/store_test.cc | 70 + ceph/src/test/pybind/CMakeLists.txt | 6 +- ceph/src/test/pybind/test_ceph_argparse.py | 74 +- ceph/src/test/pybind/test_cephfs.py | 16 + ceph/src/test/rgw/test_cls_fifo_legacy.cc | 216 +- ceph/src/test/rgw/test_log_backing.cc | 64 +- ceph/src/test/rgw/test_rgw_lua.cc | 4 +- ceph/src/test/rgw/test_rgw_manifest.cc | 33 +- ceph/src/test/rgw/test_rgw_period_history.cc | 30 +- ceph/src/test/rgw/test_rgw_throttle.cc | 6 +- ceph/src/test/test_cors.cc | 3 +- ceph/src/test/test_mempool.cc | 17 +- ceph/src/tools/cephfs/DataScan.cc | 11 +- ceph/src/tools/cephfs/top/cephfs-top | 212 +- ceph/src/tools/cephfs_mirror/FSMirror.cc | 13 +- ceph/src/tools/cephfs_mirror/FSMirror.h | 2 + ceph/src/tools/cephfs_mirror/Mirror.cc | 24 +- ceph/src/tools/cephfs_mirror/Mirror.h | 2 + ceph/src/tools/cephfs_mirror/PeerReplayer.cc | 837 ++- ceph/src/tools/cephfs_mirror/PeerReplayer.h | 141 +- ceph/src/tools/cephfs_mirror/Utils.cc | 31 +- ceph/src/tools/cephfs_mirror/Utils.h | 3 +- ceph/src/tools/cephfs_mirror/main.cc | 6 +- ceph/src/tools/rbd/action/Migration.cc | 4 +- ceph/src/tools/rbd/action/Status.cc | 64 +- ceph/src/tools/rbd/action/Trash.cc | 17 +- .../image_replayer/snapshot/Replayer.h | 2 +- ceph/src/vstart.sh | 12 +- ceph/win32_build.sh | 18 +- ceph/win32_deps_build.sh | 134 +- 861 files changed, 29224 insertions(+), 15161 deletions(-) create mode 100644 ceph/debian/libsqlite3-mod-ceph.symbols create mode 100644 ceph/doc/cephfs/cephfs-mirroring.rst create mode 100644 ceph/doc/rados/configuration/mclock-config-ref.rst create mode 100644 ceph/qa/cephfs/clusters/1-node-1-mds-1-osd.yaml create mode 100644 ceph/qa/distros/all/rhel_8.4.yaml create mode 120000 ceph/qa/suites/fs/bugs/client_trim_caps/centos_latest.yaml rename ceph/qa/suites/{rados/cephadm/dashboard => fs/full}/% (100%) rename ceph/qa/suites/{rados/cephadm/thrash-old-clients => fs/full}/.qa (100%) create mode 120000 ceph/qa/suites/fs/full/begin.yaml rename ceph/qa/suites/{rados/cephadm/thrash-old-clients/0-size-min-size-overrides => fs/full/clusters}/.qa (100%) create mode 120000 ceph/qa/suites/fs/full/clusters/1-node-1-mds-1-osd.yaml create mode 120000 ceph/qa/suites/fs/full/conf create mode 120000 ceph/qa/suites/fs/full/distro create mode 100644 ceph/qa/suites/fs/full/mount/fuse.yaml rename ceph/qa/suites/{rados/cephadm/thrash-old-clients/1-install => fs/full/objectstore}/.qa (100%) create mode 120000 ceph/qa/suites/fs/full/objectstore/bluestore-bitmap.yaml create mode 100644 ceph/qa/suites/fs/full/overrides.yaml rename ceph/qa/suites/{rados/cephadm/thrash-old-clients/clusters => fs/full/overrides}/+ (100%) rename ceph/qa/suites/{rados/cephadm/thrash-old-clients/backoff => fs/full/overrides}/.qa (100%) create mode 120000 ceph/qa/suites/fs/full/overrides/frag_enable.yaml create mode 120000 ceph/qa/suites/fs/full/overrides/no_client_pidfile.yaml create mode 120000 ceph/qa/suites/fs/full/overrides/whitelist_health.yaml create mode 120000 ceph/qa/suites/fs/full/overrides/whitelist_wrongly_marked_down.yaml rename ceph/qa/suites/{rados/cephadm/thrash-old-clients/clusters => fs/full/tasks}/.qa (100%) create mode 100644 ceph/qa/suites/fs/full/tasks/mgr-osd-full.yaml mode change 120000 => 100644 ceph/qa/suites/fs/mirror/overrides/whitelist_health.yaml create mode 120000 ceph/qa/suites/fs/upgrade/featureful_client/old_client/centos_latest.yaml create mode 120000 ceph/qa/suites/fs/upgrade/featureful_client/upgraded_client/centos_latest.yaml mode change 120000 => 100644 ceph/qa/suites/fs/valgrind/mirror/overrides/whitelist_health.yaml delete mode 120000 ceph/qa/suites/fs/verify/distro~HEAD rename ceph/qa/suites/{rados/cephadm/orchestrator_cli/% => fs/workload/wsync/$} (100%) create mode 100644 ceph/qa/suites/fs/workload/wsync/no.yaml create mode 100644 ceph/qa/suites/fs/workload/wsync/yes.yaml rename ceph/qa/suites/{rados/cephadm => orch}/.qa (100%) rename ceph/qa/suites/{rados/cephadm/dashboard => orch/cephadm}/.qa (100%) rename ceph/qa/suites/{rados/cephadm/smoke-roleless => orch/cephadm/dashboard}/% (100%) rename ceph/qa/suites/{rados/cephadm/dashboard/0-distro => orch/cephadm/dashboard}/.qa (100%) rename ceph/qa/suites/{rados/cephadm/orchestrator_cli => orch/cephadm/dashboard/0-distro}/.qa (100%) rename ceph/qa/suites/{rados => orch}/cephadm/dashboard/0-distro/centos_8.2_kubic_stable.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/dashboard/task/test_e2e.yaml (100%) rename ceph/qa/suites/{rados/cephadm/smoke => orch/cephadm/orchestrator_cli}/% (100%) rename ceph/qa/suites/{rados/cephadm/smoke-roleless => orch/cephadm/orchestrator_cli}/.qa (100%) rename ceph/qa/suites/{rados => orch}/cephadm/orchestrator_cli/0-random-distro$ (100%) rename ceph/qa/suites/{rados => orch}/cephadm/orchestrator_cli/2-node-mgr.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/orchestrator_cli/orchestrator_cli.yaml (100%) rename ceph/qa/suites/{rados/cephadm/thrash-old-clients => orch/cephadm/smoke-roleless}/% (100%) rename ceph/qa/suites/{rados/cephadm/smoke => orch/cephadm/smoke-roleless}/.qa (100%) rename ceph/qa/suites/{rados => orch}/cephadm/smoke-roleless/0-distro (100%) rename ceph/qa/suites/{rados => orch}/cephadm/smoke-roleless/1-start.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/smoke-roleless/2-services/basic.yaml (100%) create mode 100644 ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/client-keyring.yaml rename ceph/qa/suites/{rados => orch}/cephadm/smoke-roleless/2-services/iscsi.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/smoke-roleless/2-services/mirror.yaml (100%) create mode 100644 ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress.yaml create mode 100644 ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress2.yaml create mode 100644 ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs.yaml create mode 100644 ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs2.yaml rename ceph/qa/suites/{rados => orch}/cephadm/smoke-roleless/2-services/rgw-ingress.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/smoke-roleless/2-services/rgw.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/smoke-roleless/3-final.yaml (100%) rename ceph/qa/suites/{rados/cephadm/thrash => orch/cephadm/smoke-singlehost}/% (100%) create mode 120000 ceph/qa/suites/orch/cephadm/smoke-singlehost/0-distro$ create mode 100644 ceph/qa/suites/orch/cephadm/smoke-singlehost/1-start.yaml rename ceph/qa/suites/{rados/cephadm/thrash-old-clients/backoff/normal.yaml => orch/cephadm/smoke-singlehost/2-services/basic.yaml} (100%) create mode 100644 ceph/qa/suites/orch/cephadm/smoke-singlehost/2-services/rgw.yaml create mode 100644 ceph/qa/suites/orch/cephadm/smoke-singlehost/3-final.yaml rename ceph/qa/suites/{rados/cephadm/upgrade => orch/cephadm/smoke}/% (100%) rename ceph/qa/suites/{rados/cephadm/smoke/distro => orch/cephadm/smoke}/.qa (100%) rename ceph/qa/suites/{rados/cephadm/thrash => orch/cephadm/smoke/distro}/.qa (100%) rename ceph/qa/suites/{rados => orch}/cephadm/smoke/distro/centos_8.2_kubic_stable.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/smoke/distro/rhel_8.3_kubic_stable.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/smoke/distro/ubuntu_18.04.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/smoke/distro/ubuntu_20.04.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/smoke/fixed-2.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/smoke/mon_election (100%) rename ceph/qa/suites/{rados => orch}/cephadm/smoke/start.yaml (100%) rename ceph/qa/suites/{rados/cephadm/with-work => orch/cephadm/thrash-old-clients}/% (100%) rename ceph/qa/suites/{rados/cephadm/thrash-old-clients/d-balancer => orch/cephadm/thrash-old-clients}/.qa (100%) rename ceph/qa/suites/{rados/cephadm/thrash-old-clients/distro$ => orch/cephadm/thrash-old-clients/0-size-min-size-overrides}/.qa (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/0-size-min-size-overrides/2-size-2-min-size.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/0-size-min-size-overrides/3-size-2-min-size.yaml (100%) rename ceph/qa/suites/{rados/cephadm/thrash-old-clients/msgr-failures => orch/cephadm/thrash-old-clients/1-install}/.qa (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/1-install/luminous-v1only.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/1-install/luminous.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/1-install/mimic-v1only.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/1-install/mimic.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/1-install/nautilus-v1only.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/1-install/nautilus-v2only.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/1-install/nautilus.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/1-install/octopus.yaml (100%) rename ceph/qa/suites/{rados/cephadm/thrash-old-clients/thrashers => orch/cephadm/thrash-old-clients/backoff}/.qa (100%) rename ceph/qa/suites/{rados/cephadm/thrash-old-clients/d-balancer/on.yaml => orch/cephadm/thrash-old-clients/backoff/normal.yaml} (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/backoff/peering.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/backoff/peering_and_degraded.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/ceph.yaml (100%) rename ceph/qa/suites/{rados/cephadm/thrash-old-clients/thrashers/none.yaml => orch/cephadm/thrash-old-clients/clusters/+} (100%) rename ceph/qa/suites/{rados/cephadm/thrash-old-clients/workloads => orch/cephadm/thrash-old-clients/clusters}/.qa (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/clusters/openstack.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/clusters/three-plus-one.yaml (100%) rename ceph/qa/suites/{rados/cephadm/thrash/3-tasks => orch/cephadm/thrash-old-clients/d-balancer}/.qa (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/d-balancer/crush-compat.yaml (100%) rename ceph/qa/suites/{rados/cephadm/upgrade/2-repo_digest/defaut.yaml => orch/cephadm/thrash-old-clients/d-balancer/on.yaml} (100%) rename ceph/qa/suites/{rados/cephadm/with-work => orch/cephadm/thrash-old-clients/distro$}/.qa (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/distro$/ubuntu_18.04.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/mon_election (100%) rename ceph/qa/suites/{rados/cephadm/with-work/mode => orch/cephadm/thrash-old-clients/msgr-failures}/.qa (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/msgr-failures/fastclose.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/msgr-failures/few.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/msgr-failures/osd-delay.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/rados.yaml (100%) rename ceph/qa/suites/{rados/cephadm/with-work/tasks => orch/cephadm/thrash-old-clients/thrashers}/.qa (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/thrashers/careful.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/thrashers/default.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/thrashers/mapgap.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/thrashers/morepggrow.yaml (100%) rename ceph/qa/suites/{rados/cephadm/workunits/% => orch/cephadm/thrash-old-clients/thrashers/none.yaml} (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/thrashers/pggrow.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/thrashosds-health.yaml (100%) create mode 120000 ceph/qa/suites/orch/cephadm/thrash-old-clients/workloads/.qa rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/workloads/cache-snaps.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/workloads/radosbench.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/workloads/rbd_cls.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/workloads/snaps-few-objects.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash-old-clients/workloads/test_rbd_api.yaml (100%) create mode 100644 ceph/qa/suites/orch/cephadm/thrash/% rename ceph/qa/suites/{rados/cephadm/upgrade => orch/cephadm/thrash}/.qa (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash/0-distro (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash/1-start.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash/2-thrash.yaml (100%) create mode 120000 ceph/qa/suites/orch/cephadm/thrash/3-tasks/.qa create mode 120000 ceph/qa/suites/orch/cephadm/thrash/3-tasks/rados_api_tests.yaml create mode 120000 ceph/qa/suites/orch/cephadm/thrash/3-tasks/radosbench.yaml create mode 120000 ceph/qa/suites/orch/cephadm/thrash/3-tasks/small-objects.yaml create mode 120000 ceph/qa/suites/orch/cephadm/thrash/3-tasks/snaps-few-objects.yaml rename ceph/qa/suites/{rados => orch}/cephadm/thrash/fixed-2.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash/msgr (100%) rename ceph/qa/suites/{rados => orch}/cephadm/thrash/root.yaml (100%) create mode 100644 ceph/qa/suites/orch/cephadm/upgrade/% rename ceph/qa/suites/{rados/cephadm/workunits => orch/cephadm/upgrade}/.qa (100%) rename ceph/qa/suites/{rados => orch}/cephadm/upgrade/1-start-distro/1-start-centos_8.3-octopus.yaml (95%) rename ceph/qa/suites/{rados => orch}/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04-15.2.9.yaml (95%) rename ceph/qa/suites/{rados => orch}/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml (95%) create mode 100644 ceph/qa/suites/orch/cephadm/upgrade/2-repo_digest/defaut.yaml rename ceph/qa/suites/{rados => orch}/cephadm/upgrade/2-repo_digest/repo_digest.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/upgrade/3-start-upgrade.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/upgrade/4-wait.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/upgrade/mon_election (100%) create mode 100644 ceph/qa/suites/orch/cephadm/with-work/% create mode 120000 ceph/qa/suites/orch/cephadm/with-work/.qa rename ceph/qa/suites/{rados => orch}/cephadm/with-work/0-distro (100%) rename ceph/qa/suites/{rados => orch}/cephadm/with-work/fixed-2.yaml (100%) create mode 120000 ceph/qa/suites/orch/cephadm/with-work/mode/.qa rename ceph/qa/suites/{rados => orch}/cephadm/with-work/mode/packaged.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/with-work/mode/root.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/with-work/mon_election (100%) rename ceph/qa/suites/{rados => orch}/cephadm/with-work/msgr (100%) rename ceph/qa/suites/{rados => orch}/cephadm/with-work/start.yaml (100%) create mode 120000 ceph/qa/suites/orch/cephadm/with-work/tasks/.qa create mode 120000 ceph/qa/suites/orch/cephadm/with-work/tasks/rados_api_tests.yaml create mode 120000 ceph/qa/suites/orch/cephadm/with-work/tasks/rados_python.yaml create mode 100644 ceph/qa/suites/orch/cephadm/workunits/% rename ceph/qa/suites/{rados/cephadm/workunits/0-distro => orch/cephadm/workunits}/.qa (100%) create mode 120000 ceph/qa/suites/orch/cephadm/workunits/0-distro/.qa rename ceph/qa/suites/{rados => orch}/cephadm/workunits/0-distro/centos_8.2_kubic_stable.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/workunits/mon_election (100%) rename ceph/qa/suites/{rados => orch}/cephadm/workunits/task/test_adoption.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/workunits/task/test_cephadm.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/workunits/task/test_cephadm_repos.yaml (100%) rename ceph/qa/suites/{rados => orch}/cephadm/workunits/task/test_orch_cli.yaml (100%) create mode 120000 ceph/qa/suites/orch/rook/.qa create mode 100644 ceph/qa/suites/orch/rook/smoke/% create mode 120000 ceph/qa/suites/orch/rook/smoke/.qa create mode 120000 ceph/qa/suites/orch/rook/smoke/0-distro/.qa create mode 100644 ceph/qa/suites/orch/rook/smoke/0-distro/ubuntu_18.04.yaml create mode 100644 ceph/qa/suites/orch/rook/smoke/0-distro/ubuntu_20.04.yaml create mode 100644 ceph/qa/suites/orch/rook/smoke/0-kubeadm.yaml create mode 100644 ceph/qa/suites/orch/rook/smoke/1-rook.yaml create mode 100644 ceph/qa/suites/orch/rook/smoke/2-workload/none.yaml create mode 100644 ceph/qa/suites/orch/rook/smoke/2-workload/radosbench.yaml create mode 100644 ceph/qa/suites/orch/rook/smoke/3-final.yaml create mode 100644 ceph/qa/suites/orch/rook/smoke/cluster/1-node.yaml create mode 100644 ceph/qa/suites/orch/rook/smoke/cluster/3-node.yaml create mode 100644 ceph/qa/suites/orch/rook/smoke/k8s/1.21.yaml create mode 100644 ceph/qa/suites/orch/rook/smoke/net/calico.yaml create mode 100644 ceph/qa/suites/orch/rook/smoke/rook/1.6.2.yaml create mode 100644 ceph/qa/suites/orch/rook/smoke/rook/master.yaml create mode 120000 ceph/qa/suites/rados/cephadm delete mode 120000 ceph/qa/suites/rados/cephadm/thrash/3-tasks/rados_api_tests.yaml delete mode 120000 ceph/qa/suites/rados/cephadm/thrash/3-tasks/radosbench.yaml delete mode 120000 ceph/qa/suites/rados/cephadm/thrash/3-tasks/small-objects.yaml delete mode 120000 ceph/qa/suites/rados/cephadm/thrash/3-tasks/snaps-few-objects.yaml delete mode 120000 ceph/qa/suites/rados/cephadm/with-work/tasks/rados_api_tests.yaml delete mode 120000 ceph/qa/suites/rados/cephadm/with-work/tasks/rados_python.yaml create mode 120000 ceph/qa/suites/rados/rook delete mode 120000 ceph/qa/suites/rados/standalone/mon_election create mode 100644 ceph/qa/suites/rados/thrash/3-scrub-overrides/$ create mode 120000 ceph/qa/suites/rados/thrash/3-scrub-overrides/.qa create mode 100644 ceph/qa/suites/rados/thrash/3-scrub-overrides/default.yaml create mode 100644 ceph/qa/suites/rados/thrash/3-scrub-overrides/max-simultaneous-scrubs-2.yaml create mode 100644 ceph/qa/suites/rados/thrash/3-scrub-overrides/max-simultaneous-scrubs-3.yaml create mode 100644 ceph/qa/suites/rados/thrash/msgr-failures/osd-dispatch-delay.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-parallel/% create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-parallel/point-to-point-upgrade.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-parallel/supported-all-distro/ubuntu_latest.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/% create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/0-cluster/+ create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/0-cluster/openstack.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/0-cluster/start.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/1-ceph-install/pacific..yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/1.1.short_pg_log.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/2-partial-upgrade/firsthalf.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/3-thrash/default.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/+ create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/fsx.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/radosbench.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/rbd-cls.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/rbd-import-export.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/rbd_api.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/readwrite.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/snaps-few-objects.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/5-finish-upgrade.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/6-final-workload/+ create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/6-final-workload/rbd-python.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/6-final-workload/snaps-many-objects.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/objectstore/bluestore-bitmap.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/objectstore/bluestore-comp.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/objectstore/bluestore-stupid.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/objectstore/filestore-xfs.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/supported-all-distro/ubuntu_latest.yaml create mode 100644 ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/thrashosds-health.yaml create mode 100644 ceph/qa/tasks/kubeadm.py create mode 100644 ceph/qa/tasks/rook-ceph.conf create mode 100644 ceph/qa/tasks/rook.py create mode 100755 ceph/qa/workunits/fs/full/subvolume_rm.sh create mode 100644 ceph/src/pybind/mgr/cephadm/autotune.py create mode 100644 ceph/src/pybind/mgr/cephadm/tests/test_autotune.py create mode 100755 ceph/src/pybind/mgr/dashboard/ci/cephadm/bootstrap-cluster.sh create mode 100755 ceph/src/pybind/mgr/dashboard/ci/cephadm/ceph_cluster.yml create mode 100755 ceph/src/pybind/mgr/dashboard/ci/cephadm/run-cephadm-e2e-tests.sh create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/fixtures/orchestrator/services.json create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/services.po.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts-force-maintenance.e2e-spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/05-services.e2e-spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/01-hosts.e2e-spec.ts rename ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/{1.0f6133f836c4310d2d34.js => 1.1c3aa698fdc6e2e06036.js} (79%) create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/5.7ef5282604f6c3e77fd4.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/5.f1a521f29ba388aefe88.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/6.5aa9ef51cf028c8fa150.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/6.93b626d621206efed808.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/main.686c64fd8301e15fd70a.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/main.abdc8ba4d5af20b6adbf.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/main.e73fdb683302952f5ca0.css rename ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/{runtime.8d55cba33eadfad4c77c.js => runtime.95ade215c318c70e8872.js} (65%) delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/scripts.38cee5fbd56812617717.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/scripts.6bda3fa7e09a87cd4228.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/styles.0b9eb5355e15caa2c87d.css create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/styles.7918cb8dc788b3eedc95.css create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/classes/css-helper.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/color.enum.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/enum/health-color.enum.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/styles.scss.d.ts create mode 100644 ceph/src/pybind/mgr/nfs/__init__.py create mode 100644 ceph/src/pybind/mgr/nfs/cluster.py create mode 100644 ceph/src/pybind/mgr/nfs/exception.py create mode 100644 ceph/src/pybind/mgr/nfs/export.py create mode 100644 ceph/src/pybind/mgr/nfs/export_utils.py create mode 100644 ceph/src/pybind/mgr/nfs/module.py create mode 100644 ceph/src/pybind/mgr/nfs/utils.py delete mode 100644 ceph/src/pybind/mgr/volumes/fs/nfs.py create mode 100644 ceph/src/python-common/ceph/tests/test_hostspec.py diff --git a/ceph/.github/labeler.yml b/ceph/.github/labeler.yml index 0821fe964..5719e113f 100644 --- a/ceph/.github/labeler.yml +++ b/ceph/.github/labeler.yml @@ -1,3 +1,6 @@ +api-change: + - src/pybind/mgr/dashboard/openapi.yaml + build/ops: - "**/CMakeLists.txt" - admin/** diff --git a/ceph/CMakeLists.txt b/ceph/CMakeLists.txt index f0ed97d1a..86b32019e 100644 --- a/ceph/CMakeLists.txt +++ b/ceph/CMakeLists.txt @@ -52,6 +52,7 @@ if(MINGW) set(CMAKE_CXX_LINK_EXECUTABLE " -o ${CMAKE_GNULD_IMAGE_VERSION} ") + link_directories(${MINGW_LINK_DIRECTORIES}) endif() option(WITH_CCACHE "Build with ccache.") @@ -716,4 +717,4 @@ add_tags(ctags EXCLUDES "*.js" "*.css" ".tox" "python-common/build") add_custom_target(tags DEPENDS ctags) -set(VERSION 16.2.4) +set(VERSION 16.2.5) diff --git a/ceph/PendingReleaseNotes b/ceph/PendingReleaseNotes index 1963b699d..88c0137c0 100644 --- a/ceph/PendingReleaseNotes +++ b/ceph/PendingReleaseNotes @@ -11,6 +11,43 @@ >=16.0.0 -------- + +* `ceph-mgr-modules-core` debian package does not recommend `ceph-mgr-rook` + anymore. As the latter depends on `python3-numpy` which cannot be imported in + different Python sub-interpreters multi-times if the version of + `python3-numpy` is older than 1.19. Since `apt-get` installs the `Recommends` + packages by default, `ceph-mgr-rook` was always installed along with + `ceph-mgr` debian package as an indirect dependency. If your workflow depends + on this behavior, you might want to install `ceph-mgr-rook` separately. + +* mgr/nfs: ``nfs`` module is moved out of volumes plugin. Prior using the + ``ceph nfs`` commands, ``nfs`` mgr module must be enabled. + +* volumes/nfs: The ``cephfs`` cluster type has been removed from the + ``nfs cluster create`` subcommand. Clusters deployed by cephadm can + support an NFS export of both ``rgw`` and ``cephfs`` from a single + NFS cluster instance. + +* The ``nfs cluster update`` command has been removed. You can modify + the placement of an existing NFS service (and/or its associated + ingress service) using ``orch ls --export`` and ``orch apply -i + ...``. + +* The ``orch apply nfs`` command no longer requires a pool or + namespace argument. We strongly encourage users to use the defaults + so that the ``nfs cluster ls`` and related commands will work + properly. + +* The ``nfs cluster delete`` and ``nfs export delete`` commands are + deprecated and will be removed in a future release. Please use + ``nfs cluster rm`` and ``nfs export rm`` instead. + +* mgr-pg_autoscaler: Autoscaler will now start out by scaling each + pool to have a full complements of pgs from the start and will only + decrease it when other pools need more pgs due to increased usage. + This improves out of the box performance of Ceph by allowing more PGs + to be created for a given pool. + * CephFS: Disabling allow_standby_replay on a file system will also stop all standby-replay daemons for that file system. diff --git a/ceph/ceph.spec b/ceph/ceph.spec index 440563f6b..2a07bdb6e 100644 --- a/ceph/ceph.spec +++ b/ceph/ceph.spec @@ -122,7 +122,7 @@ # main package definition ################################################################################# Name: ceph -Version: 16.2.4 +Version: 16.2.5 Release: 0%{?dist} %if 0%{?fedora} || 0%{?rhel} Epoch: 2 @@ -138,7 +138,7 @@ License: LGPL-2.1 and LGPL-3.0 and CC-BY-SA-3.0 and GPL-2.0 and BSL-1.0 and BSD- Group: System/Filesystems %endif URL: http://ceph.com/ -Source0: %{?_remote_tarball_prefix}ceph-16.2.4.tar.bz2 +Source0: %{?_remote_tarball_prefix}ceph-16.2.5.tar.bz2 %if 0%{?suse_version} # _insert_obs_source_lines_here ExclusiveArch: x86_64 aarch64 ppc64le s390x @@ -1205,7 +1205,7 @@ This package provides Ceph default alerts for Prometheus. # common ################################################################################# %prep -%autosetup -p1 -n ceph-16.2.4 +%autosetup -p1 -n ceph-16.2.5 %build # LTO can be enabled as soon as the following GCC bug is fixed: @@ -1767,6 +1767,7 @@ fi %{_datadir}/ceph/mgr/localpool %{_datadir}/ceph/mgr/mds_autoscaler %{_datadir}/ceph/mgr/mirroring +%{_datadir}/ceph/mgr/nfs %{_datadir}/ceph/mgr/orchestrator %{_datadir}/ceph/mgr/osd_perf_query %{_datadir}/ceph/mgr/osd_support diff --git a/ceph/ceph.spec.in b/ceph/ceph.spec.in index 718421ca9..5e2ffec7d 100644 --- a/ceph/ceph.spec.in +++ b/ceph/ceph.spec.in @@ -1767,6 +1767,7 @@ fi %{_datadir}/ceph/mgr/localpool %{_datadir}/ceph/mgr/mds_autoscaler %{_datadir}/ceph/mgr/mirroring +%{_datadir}/ceph/mgr/nfs %{_datadir}/ceph/mgr/orchestrator %{_datadir}/ceph/mgr/osd_perf_query %{_datadir}/ceph/mgr/osd_support diff --git a/ceph/changelog.upstream b/ceph/changelog.upstream index 28cda449b..1bf5eab85 100644 --- a/ceph/changelog.upstream +++ b/ceph/changelog.upstream @@ -1,7 +1,13 @@ -ceph (16.2.4-1focal) focal; urgency=medium +ceph (16.2.5-1focal) focal; urgency=medium - -- Jenkins Build Slave User Thu, 13 May 2021 17:30:29 +0000 + -- Jenkins Build Slave User Thu, 08 Jul 2021 14:16:59 +0000 + +ceph (16.2.5-1) stable; urgency=medium + + * New upstream release + + -- Ceph Release Team Thu, 08 Jul 2021 14:03:54 +0000 ceph (16.2.4-1) stable; urgency=medium diff --git a/ceph/debian/ceph-mgr-modules-core.install b/ceph/debian/ceph-mgr-modules-core.install index 764038dd0..e99f78efb 100644 --- a/ceph/debian/ceph-mgr-modules-core.install +++ b/ceph/debian/ceph-mgr-modules-core.install @@ -7,6 +7,7 @@ usr/share/ceph/mgr/insights usr/share/ceph/mgr/iostat usr/share/ceph/mgr/localpool usr/share/ceph/mgr/mirroring +usr/share/ceph/mgr/nfs usr/share/ceph/mgr/orchestrator usr/share/ceph/mgr/osd_perf_query usr/share/ceph/mgr/osd_support diff --git a/ceph/debian/control b/ceph/debian/control index 4ca0caa76..a319e5a82 100644 --- a/ceph/debian/control +++ b/ceph/debian/control @@ -226,7 +226,6 @@ Package: ceph-mgr Architecture: linux-any Depends: ceph-base (= ${binary:Version}), ceph-mgr-modules-core (= ${binary:Version}), - libsqlite3-mod-ceph, python3-bcrypt, python3-cherrypy3, python3-distutils, @@ -302,7 +301,6 @@ Depends: ${misc:Depends}, python3-openssl, Replaces: ceph-mgr (<< 15.1.0) Breaks: ceph-mgr (<< 15.1.0) -Recommends: ceph-mgr-rook Description: ceph manager modules which are always enabled Ceph is a massively scalable, open-source, distributed storage system that runs on commodity hardware and delivers object, diff --git a/ceph/debian/libsqlite3-mod-ceph.symbols b/ceph/debian/libsqlite3-mod-ceph.symbols new file mode 100644 index 000000000..d9679a13d --- /dev/null +++ b/ceph/debian/libsqlite3-mod-ceph.symbols @@ -0,0 +1,37 @@ +libcephsqlite.so libsqlite3-mod-ceph #MINVER# + _ZGVN18SimpleRADOSStriper7biglockB5cxx11E@Base 15.2.0-1 + _ZGVN18SimpleRADOSStriper8lockdescB5cxx11E@Base 15.2.0-1 + _ZN18SimpleRADOSStriper10XATTR_EXCLE@Base 15.2.0-1 + _ZN18SimpleRADOSStriper10XATTR_SIZEE@Base 15.2.0-1 + _ZN18SimpleRADOSStriper12recover_lockEv@Base 15.2.0-1 + _ZN18SimpleRADOSStriper12set_metadataEmb@Base 15.2.0-1 + _ZN18SimpleRADOSStriper12shrink_allocEm@Base 15.2.0-1 + _ZN18SimpleRADOSStriper13XATTR_VERSIONE@Base 15.2.0-1 + _ZN18SimpleRADOSStriper13config_loggerEPN4ceph6common11CephContextESt17basic_string_viewIcSt11char_traitsIcEEPSt10shared_ptrINS1_12PerfCountersEE@Base 15.2.0-1 + _ZN18SimpleRADOSStriper13print_lockersERSo@Base 15.2.0-1 + _ZN18SimpleRADOSStriper13wait_for_aiosEb@Base 15.2.0-1 + _ZN18SimpleRADOSStriper15XATTR_ALLOCATEDE@Base 15.2.0-1 + _ZN18SimpleRADOSStriper16lock_keeper_mainEv@Base 15.2.0-1 + _ZN18SimpleRADOSStriper18maybe_shrink_allocEv@Base 15.2.0-1 + _ZN18SimpleRADOSStriper24XATTR_LAYOUT_OBJECT_SIZEE@Base 15.2.0-1 + _ZN18SimpleRADOSStriper24XATTR_LAYOUT_STRIPE_UNITE@Base 15.2.0-1 + _ZN18SimpleRADOSStriper25XATTR_LAYOUT_STRIPE_COUNTE@Base 15.2.0-1 + _ZN18SimpleRADOSStriper4lockEm@Base 15.2.0-1 + _ZN18SimpleRADOSStriper4openEv@Base 15.2.0-1 + _ZN18SimpleRADOSStriper4readEPvmm@Base 15.2.0-1 + _ZN18SimpleRADOSStriper4statEPm@Base 15.2.0-1 + _ZN18SimpleRADOSStriper5flushEv@Base 15.2.0-1 + _ZN18SimpleRADOSStriper5writeEPKvmm@Base 15.2.0-1 + _ZN18SimpleRADOSStriper6createEv@Base 15.2.0-1 + _ZN18SimpleRADOSStriper6removeEv@Base 15.2.0-1 + _ZN18SimpleRADOSStriper6str2blESt17basic_string_viewIcSt11char_traitsIcEE@Base 15.2.0-1 + _ZN18SimpleRADOSStriper6unlockEv@Base 15.2.0-1 + _ZN18SimpleRADOSStriper7biglockB5cxx11E@Base 15.2.0-1 + _ZN18SimpleRADOSStriper7uint2blEm@Base 15.2.0-1 + _ZN18SimpleRADOSStriper8lockdescB5cxx11E@Base 15.2.0-1 + _ZN18SimpleRADOSStriper8truncateEm@Base 15.2.0-1 + _ZN18SimpleRADOSStriperD1Ev@Base 15.2.0-1 + _ZN18SimpleRADOSStriperD2Ev@Base 15.2.0-1 + _ZNK18SimpleRADOSStriper15get_next_extentEmm@Base 15.2.0-1 + cephsqlite_setcct@Base 15.2.0-1 + sqlite3_cephsqlite_init@Base 15.2.0-1 diff --git a/ceph/doc/cephadm/adoption.rst b/ceph/doc/cephadm/adoption.rst index 61648fd9e..db4fded39 100644 --- a/ceph/doc/cephadm/adoption.rst +++ b/ceph/doc/cephadm/adoption.rst @@ -126,8 +126,8 @@ Adoption process This will perform a ``cephadm check-host`` on each host before adding it; this check ensures that the host is functioning properly. The IP address - argument is required only if DNS does not allow you to connect to each host - by its short name. + argument is recommended; if not provided, then the host name will be resolved + via DNS. #. Verify that the adopted monitor and manager daemons are visible: diff --git a/ceph/doc/cephadm/compatibility.rst b/ceph/doc/cephadm/compatibility.rst index a616506de..7c75b7445 100644 --- a/ceph/doc/cephadm/compatibility.rst +++ b/ceph/doc/cephadm/compatibility.rst @@ -3,6 +3,8 @@ Compatibility and Stability =========================== +.. _cephadm-compatibility-with-podman: + Compatibility with Podman Versions ---------------------------------- @@ -25,6 +27,10 @@ Those versions are expected to work: | >= 16.2.1 | False | True | True | False | True | +-----------+-------+-------+-------+-------+-------+ +.. warning:: + Only podman versions that are 2.0.0 and higher work with Ceph Pacific, with the exception of podman version 2.2.1, which does not work with Ceph Pacific. kubic stable is known to work with Ceph Pacific, but it must be run with a newer kernel. + + .. _cephadm-stability: Stability diff --git a/ceph/doc/cephadm/host-management.rst b/ceph/doc/cephadm/host-management.rst index edf6772d4..bcf626752 100644 --- a/ceph/doc/cephadm/host-management.rst +++ b/ceph/doc/cephadm/host-management.rst @@ -37,15 +37,28 @@ To add each new host to the cluster, perform two steps: .. prompt:: bash # - ceph orch host add *newhost* + ceph orch host add ** [**] [* ...*] For example: .. prompt:: bash # - ceph orch host add host2 - ceph orch host add host3 - + ceph orch host add host2 10.10.0.102 + ceph orch host add host3 10.10.0.103 + + It is best to explicitly provide the host IP address. If an IP is + not provided, then the host name will be immediately resolved via + DNS and that IP will be used. + + One or more labels can also be included to immediately label the + new host. For example, by default the ``_admin`` label will make + cephadm maintain a copy of the ``ceph.conf`` file and a + ``client.admin`` keyring file in ``/etc/ceph``: + + .. prompt:: bash # + + ceph orch host add host4 10.10.0.104 --labels _admin + .. _cephadm-removing-hosts: Removing Hosts @@ -117,7 +130,34 @@ To remove a label, run:: ceph orch host label rm my_hostname my_label - + +.. _cephadm-special-host-labels: + +Special host labels +------------------- + +The following host labels have a special meaning to cephadm. All start with ``_``. + +* ``_no_schedule``: *Do not schedule or deploy daemons on this host*. + + This label prevents cephadm from deploying daemons on this host. If it is added to + an existing host that already contains Ceph daemons, it will cause cephadm to move + those daemons elsewhere (except OSDs, which are not removed automatically). + +* ``_no_autotune_memory``: *Do not autotune memory on this host*. + + This label will prevent daemon memory from being tuned even when the + ``osd_memory_target_autotune`` or similar option is enabled for one or more daemons + on that host. + +* ``_admin``: *Distribute client.admin and ceph.conf to this host*. + + By default, an ``_admin`` label is applied to the first host in the cluster (where + bootstrap was originally run), and the ``client.admin`` key is set to be distributed + to that host via the ``ceph orch client-keyring ...`` function. Adding this label + to additional hosts will normally cause cephadm to deploy config and keyring files + in ``/etc/ceph``. + Maintenance Mode ================ @@ -138,21 +178,21 @@ Many hosts can be added at once using --- service_type: host - addr: node-00 hostname: node-00 + addr: 192.168.0.10 labels: - example1 - example2 --- service_type: host - addr: node-01 hostname: node-01 + addr: 192.168.0.11 labels: - grafana --- service_type: host - addr: node-02 hostname: node-02 + addr: 192.168.0.12 This can be combined with service specifications (below) to create a cluster spec file to deploy a whole cluster in one command. see ``cephadm bootstrap --apply-spec`` @@ -250,24 +290,12 @@ There are two ways to customize this configuration for your environment: Fully qualified domain names vs bare host names =============================================== -cephadm has very minimal requirements when it comes to resolving host -names etc. When cephadm initiates an ssh connection to a remote host, -the host name can be resolved in four different ways: - -- a custom ssh config resolving the name to an IP -- via an externally maintained ``/etc/hosts`` -- via explicitly providing an IP address to cephadm: ``ceph orch host add `` -- automatic name resolution via DNS. - -Ceph itself uses the command ``hostname`` to determine the name of the -current host. - .. note:: cephadm demands that the name of the host given via ``ceph orch host add`` equals the output of ``hostname`` on remote hosts. -Otherwise cephadm can't be sure, the host names returned by +Otherwise cephadm can't be sure that names returned by ``ceph * metadata`` match the hosts known to cephadm. This might result in a :ref:`cephadm-stray-host` warning. diff --git a/ceph/doc/cephadm/install.rst b/ceph/doc/cephadm/install.rst index 244c08b85..2a4a8887a 100644 --- a/ceph/doc/cephadm/install.rst +++ b/ceph/doc/cephadm/install.rst @@ -23,6 +23,13 @@ Requirements Any modern Linux distribution should be sufficient. Dependencies are installed automatically by the bootstrap process below. +See the section :ref:`Compatibility With Podman +Versions` for a table of Ceph versions that +are compatible with Podman. Not every version of Podman is compatible with +Ceph. + + + .. _get-cephadm: Install cephadm @@ -66,8 +73,8 @@ curl-based installation * Although the standalone script is sufficient to get a cluster started, it is convenient to have the ``cephadm`` command installed on the host. To install - the packages that provide the ``cephadm`` command for the Octopus release, - run the following commands: + the packages that provide the ``cephadm`` command, run the following + commands: .. prompt:: bash # :substitutions: @@ -148,11 +155,14 @@ This command will: host. * Generate a new SSH key for the Ceph cluster and add it to the root user's ``/root/.ssh/authorized_keys`` file. +* Write a copy of the public key to ``/etc/ceph/ceph.pub``. * Write a minimal configuration file to ``/etc/ceph/ceph.conf``. This file is needed to communicate with the new cluster. * Write a copy of the ``client.admin`` administrative (privileged!) secret key to ``/etc/ceph/ceph.client.admin.keyring``. -* Write a copy of the public key to ``/etc/ceph/ceph.pub``. +* Add the ``_admin`` label to the bootstrap host. By default, any host + with this label will (also) get a copy of ``/etc/ceph/ceph.conf`` and + ``/etc/ceph/ceph.client.admin.keyring``. Further information about cephadm bootstrap ------------------------------------------- @@ -184,7 +194,13 @@ available options. * You can pass any initial Ceph configuration options to the new cluster by putting them in a standard ini-style configuration file - and using the ``--config **`` option. + and using the ``--config **`` option. For example:: + + $ cat < initial-ceph.conf + [global] + osd crush chooseleaf type = 0 + EOF + $ ./cephadm bootstrap --config initial-ceph.conf ... * The ``--ssh-user **`` option makes it possible to choose which ssh user cephadm will use to connect to hosts. The associated ssh key will be @@ -266,6 +282,16 @@ Adding Hosts Next, add all hosts to the cluster by following :ref:`cephadm-adding-hosts`. +By default, a ``ceph.conf`` file and a copy of the ``client.admin`` keyring +are maintained in ``/etc/ceph`` on all hosts with the ``_admin`` label, which is initially +applied only to the bootstrap host. We usually recommend that one or more other hosts be +given the ``_admin`` label so that the Ceph CLI (e.g., via ``cephadm shell``) is easily +accessible on multiple hosts. To add the ``_admin`` label to additional host(s), + + .. prompt:: bash # + + ceph orch host label add ** _admin + Adding additional MONs ====================== diff --git a/ceph/doc/cephadm/monitoring.rst b/ceph/doc/cephadm/monitoring.rst index e2e180c22..14593b1bb 100644 --- a/ceph/doc/cephadm/monitoring.rst +++ b/ceph/doc/cephadm/monitoring.rst @@ -41,59 +41,75 @@ Manager `_ and `Grafana Deploying monitoring with cephadm --------------------------------- -By default, bootstrap will deploy a basic monitoring stack. If you -did not do this (by passing ``--skip-monitoring-stack``, or if you -converted an existing cluster to cephadm management, you can set up -monitoring by following the steps below. +The default behavior of ``cephadm`` is to deploy a basic monitoring stack. It +is however possible that you have a Ceph cluster without a monitoring stack, +and you would like to add a monitoring stack to it. (Here are some ways that +you might have come to have a Ceph cluster without a monitoring stack: You +might have passed the ``--skip-monitoring stack`` option to ``cephadm`` during +the installation of the cluster, or you might have converted an existing +cluster (which had no monitoring stack) to cephadm management.) -#. Enable the prometheus module in the ceph-mgr daemon. This exposes the internal Ceph metrics so that prometheus can scrape them. +To set up monitoring on a Ceph cluster that has no monitoring, follow the +steps below: - .. code-block:: bash +#. Enable the Prometheus module in the ceph-mgr daemon. This exposes the internal Ceph metrics so that Prometheus can scrape them: + + .. prompt:: bash # ceph mgr module enable prometheus -#. Deploy a node-exporter service on every node of the cluster. The node-exporter provides host-level metrics like CPU and memory utilization. +#. Deploy a node-exporter service on every node of the cluster. The node-exporter provides host-level metrics like CPU and memory utilization: - .. code-block:: bash + .. prompt:: bash # ceph orch apply node-exporter '*' -#. Deploy alertmanager +#. Deploy alertmanager: - .. code-block:: bash + .. prompt:: bash # ceph orch apply alertmanager 1 -#. Deploy prometheus. A single prometheus instance is sufficient, but - for HA you may want to deploy two. +#. Deploy Prometheus. A single Prometheus instance is sufficient, but + for high availablility (HA) you might want to deploy two: + + .. prompt:: bash # + + ceph orch apply prometheus 1 - .. code-block:: bash + or - ceph orch apply prometheus 1 # or 2 + .. prompt:: bash # + + ceph orch apply prometheus 2 -#. Deploy grafana +#. Deploy grafana: - .. code-block:: bash + .. prompt:: bash # ceph orch apply grafana 1 -Cephadm takes care of the configuration of Prometheus, Grafana, and Alertmanager -automatically. +Manually setting the Grafana URL +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Cephadm automatically configures Prometheus, Grafana, and Alertmanager in +all cases except one. -However, there is one exception to this rule. In a some setups, the Dashboard -user's browser might not be able to access the Grafana URL configured in Ceph -Dashboard. One such scenario is when the cluster and the accessing user are each -in a different DNS zone. +In a some setups, the Dashboard user's browser might not be able to access the +Grafana URL that is configured in Ceph Dashboard. This can happen when the +cluster and the accessing user are in different DNS zones. -For this case, there is an extra configuration option for Ceph Dashboard, which -can be used to configure the URL for accessing Grafana by the user's browser. -This value will never be altered by cephadm. To set this configuration option, -issue the following command:: +If this is the case, you can use a configuration option for Ceph Dashboard +to set the URL that the user's browser will use to access Grafana. This +value will never be altered by cephadm. To set this configuration option, +issue the following command: - $ ceph dashboard set-grafana-frontend-api-url + .. prompt:: bash $ -It may take a minute or two for services to be deployed. Once -completed, you should see something like this from ``ceph orch ls`` + ceph dashboard set-grafana-frontend-api-url + +It might take a minute or two for services to be deployed. After the +services have been deployed, you should see something like this when you issue the command ``ceph orch ls``: .. code-block:: console @@ -108,26 +124,43 @@ completed, you should see something like this from ``ceph orch ls`` Configuring SSL/TLS for Grafana ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``cephadm`` will deploy Grafana using the certificate defined in the ceph -key/value store. If a certificate is not specified, ``cephadm`` will generate a -self-signed certificate during deployment of the Grafana service. +``cephadm`` deploys Grafana using the certificate defined in the ceph +key/value store. If no certificate is specified, ``cephadm`` generates a +self-signed certificate during the deployment of the Grafana service. -A custom certificate can be configured using the following commands. +A custom certificate can be configured using the following commands: -.. code-block:: bash +.. prompt:: bash # ceph config-key set mgr/cephadm/grafana_key -i $PWD/key.pem ceph config-key set mgr/cephadm/grafana_crt -i $PWD/certificate.pem -If you already deployed Grafana, you need to ``reconfig`` the service for the -configuration to be updated. +If you have already deployed Grafana, run ``reconfig`` on the service to +update its configuration: -.. code-block:: bash +.. prompt:: bash # ceph orch reconfig grafana -The ``reconfig`` command also takes care of setting the right URL for Ceph -Dashboard. +The ``reconfig`` command also sets the proper URL for Ceph Dashboard. + +Networks and Ports +~~~~~~~~~~~~~~~~~~ + +All monitoring services can have the network and port they bind to configured with a yaml service specification + +example spec file: + +.. code-block:: yaml + + service_type: grafana + service_name: grafana + placement: + count: 1 + networks: + - 192.169.142.0/24 + spec: + port: 4200 Using custom images ~~~~~~~~~~~~~~~~~~~ @@ -153,6 +186,17 @@ For example ceph config set mgr mgr/cephadm/container_image_prometheus prom/prometheus:v1.4.1 +If there were already running monitoring stack daemon(s) of the type whose +image you've changed, you must redeploy the daemon(s) in order to have them +actually use the new image. + +For example, if you had changed the prometheus image + +.. prompt:: bash # + + ceph orch redeploy prometheus + + .. note:: By setting a custom image, the default value will be overridden (but not @@ -256,16 +300,15 @@ Example Disabling monitoring -------------------- -If you have deployed monitoring and would like to remove it, you can do -so with +To disable monitoring and remove the software that supports it, run the following commands: -.. code-block:: bash +.. code-block:: console - ceph orch rm grafana - ceph orch rm prometheus --force # this will delete metrics data collected so far - ceph orch rm node-exporter - ceph orch rm alertmanager - ceph mgr module disable prometheus + $ ceph orch rm grafana + $ ceph orch rm prometheus --force # this will delete metrics data collected so far + $ ceph orch rm node-exporter + $ ceph orch rm alertmanager + $ ceph mgr module disable prometheus Deploying monitoring manually diff --git a/ceph/doc/cephadm/nfs.rst b/ceph/doc/cephadm/nfs.rst index 51da9f084..631fe51bd 100644 --- a/ceph/doc/cephadm/nfs.rst +++ b/ceph/doc/cephadm/nfs.rst @@ -1,42 +1,43 @@ +.. _deploy-cephadm-nfs-ganesha: + =========== NFS Service =========== .. note:: Only the NFSv4 protocol is supported. -.. _deploy-cephadm-nfs-ganesha: +The simplest way to manage NFS is via the ``ceph nfs cluster ...`` +commands; see :ref:`cephfs-nfs`. This document covers how to manage the +cephadm services directly, which should only be necessary for unusual NFS +configurations. Deploying NFS ganesha ===================== -Cephadm deploys NFS Ganesha using a pre-defined RADOS *pool* -and optional *namespace* +Cephadm deploys NFS Ganesha daemon (or set of daemons). The configuration for +NFS is stored in the ``nfs-ganesha`` pool and exports are managed via the +``ceph nfs export ...`` commands and via the dashboard. To deploy a NFS Ganesha gateway, run the following command: .. prompt:: bash # - ceph orch apply nfs ** ** ** --placement="** [** ...]" + ceph orch apply nfs ** [--port **] [--placement ...] -For example, to deploy NFS with a service id of *foo*, that will use the RADOS -pool *nfs-ganesha* and namespace *nfs-ns*: +For example, to deploy NFS with a service id of *foo* on the default +port 2049 with the default placement of a single daemon: .. prompt:: bash # - ceph orch apply nfs foo nfs-ganesha nfs-ns - -.. note:: - Create the *nfs-ganesha* pool first if it doesn't exist. + ceph orch apply nfs foo -See :ref:`orchestrator-cli-placement-spec` for details of the placement specification. +See :ref:`orchestrator-cli-placement-spec` for the details of the placement +specification. Service Specification ===================== -Alternatively, an NFS service can also be applied using a YAML specification. - -A service of type ``nfs`` requires a pool name and may contain -an optional namespace: +Alternatively, an NFS service can be applied using a YAML specification. .. code-block:: yaml @@ -47,15 +48,66 @@ an optional namespace: - host1 - host2 spec: - pool: mypool - namespace: mynamespace + port: 12345 -where ``pool`` is a RADOS pool where NFS client recovery data is stored -and ``namespace`` is a RADOS namespace where NFS client recovery -data is stored in the pool. +In this example, we run the server on the non-default ``port`` of +12345 (instead of the default 2049) on ``host1`` and ``host2``. -The specification can then be applied using: +The specification can then be applied by running the following command: .. prompt:: bash # ceph orch apply -i nfs.yaml + + +High-availability NFS +===================== + +Deploying an *ingress* service for an existing *nfs* service will provide: + +* a stable, virtual IP that can be used to access the NFS server +* fail-over between hosts if there is a host failure +* load distribution across multiple NFS gateways (although this is rarely necessary) + +Ingress for NFS can be deployed for an existing NFS service +(``nfs.mynfs`` in this example) with the following specification: + +.. code-block:: yaml + + service_type: ingress + service_id: nfs.mynfs + placement: + count: 2 + spec: + backend_service: nfs.mynfs + frontend_port: 2049 + monitor_port: 9000 + virtual_ip: 10.0.0.123/24 + +A few notes: + + * The *virtual_ip* must include a CIDR prefix length, as in the + example above. The virtual IP will normally be configured on the + first identified network interface that has an existing IP in the + same subnet. You can also specify a *virtual_interface_networks* + property to match against IPs in other networks; see + :ref:`ingress-virtual-ip` for more information. + * The *monitor_port* is used to access the haproxy load status + page. The user is ``admin`` by default, but can be modified by + via an *admin* property in the spec. If a password is not + specified via a *password* property in the spec, the auto-generated password + can be found with: + + .. prompt:: bash # + + ceph config-key get mgr/cephadm/ingress.*{svc_id}*/monitor_password + + For example: + + .. prompt:: bash # + + ceph config-key get mgr/cephadm/ingress.nfs.myfoo/monitor_password + + * The backend service (``nfs.mynfs`` in this example) should include + a *port* property that is not 2049 to avoid conflicting with the + ingress service, which could be placed on the same host(s). diff --git a/ceph/doc/cephadm/operations.rst b/ceph/doc/cephadm/operations.rst index da8e14999..56d1146de 100644 --- a/ceph/doc/cephadm/operations.rst +++ b/ceph/doc/cephadm/operations.rst @@ -299,35 +299,74 @@ CEPHADM_CHECK_KERNEL_VERSION The OS kernel version (maj.min) is checked for consistency across the hosts. Once again, the majority of the hosts is used as the basis of identifying anomalies. -/etc/ceph/ceph.conf -=================== +Client keyrings and configs +=========================== + +Cephadm can distribute copies of the ``ceph.conf`` and client keyring +files to hosts. For example, it is usually a good idea to store a +copy of the config and ``client.admin`` keyring on any hosts that will +be used to administer the cluster via the CLI. By default, cephadm will do +this for any nodes with the ``_admin`` label (which normally includes the bootstrap +host). + +When a client keyring is placed under management, cephadm will: + + - build a list of target hosts based on the specified placement spec (see :ref:`orchestrator-cli-placement-spec`) + - store a copy of the ``/etc/ceph/ceph.conf`` file on the specified host(s) + - store a copy of the keyring file on the specified host(s) + - update the ``ceph.conf`` file as needed (e.g., due to a change in the cluster monitors) + - update the keyring file if the entity's key is changed (e.g., via ``ceph auth ...`` commands) + - ensure the keyring file has the specified ownership and mode + - remove the keyring file when client keyring management is disabled + - remove the keyring file from old hosts if the keyring placement spec is updated (as needed) + +To view which client keyrings are currently under management:: + + ceph orch client-keyring ls + +To place a keyring under management:: -Cephadm distributes a minimized ``ceph.conf`` that only contains -a minimal set of information to connect to the Ceph cluster. + ceph orch client-keyring set [--mode=] [--owner=.] [--path=] -To update the configuration settings, instead of manually editing -the ``ceph.conf`` file, use the config database instead:: +- By default, the *path* will be ``/etc/ceph/client.{entity}.keyring``, which is where + Ceph looks by default. Be careful specifying alternate locations as existing files + may be overwritten. +- A placement of ``*`` (all hosts) is common. +- The mode defaults to ``0600`` and ownership to ``0:0`` (user root, group root). - ceph config set ... +For example, to create and deploy a ``client.rbd`` key to hosts with the ``rbd-client`` label and group readable by uid/gid 107 (qemu),:: -See :ref:`ceph-conf-database` for details. + ceph auth get-or-create-key client.rbd mon 'profile rbd' mgr 'profile rbd' osd 'profile rbd pool=my_rbd_pool' + ceph orch client-keyring set client.rbd label:rbd-client --owner 107:107 --mode 640 -By default, cephadm does not deploy that minimized ``ceph.conf`` across the -cluster. To enable the management of ``/etc/ceph/ceph.conf`` files on all -hosts, please enable this by running:: +The resulting keyring file is:: + + -rw-r-----. 1 qemu qemu 156 Apr 21 08:47 /etc/ceph/client.client.rbd.keyring + +To disable management of a keyring file:: + + ceph orch client-keyring rm + +Note that this will delete any keyring files for this entity that were previously written +to cluster nodes. + + +/etc/ceph/ceph.conf +=================== - ceph config set mgr mgr/cephadm/manage_etc_ceph_ceph_conf true +It may also be useful to distribute ``ceph.conf`` files to hosts without an associated +client keyring file. By default, cephadm only deploys a ``ceph.conf`` file to hosts where a client keyring +is also distributed (see above). To write config files to hosts without client keyrings:: -To set up an initial configuration before bootstrapping -the cluster, create an initial ``ceph.conf`` file. For example:: + ceph config set mgr mgr/cephadm/manage_etc_ceph_ceph_conf true - cat < /etc/ceph/ceph.conf - [global] - osd crush chooseleaf type = 0 - EOF +By default, the configs are written to all hosts (i.e., those listed +by ``ceph orch host ls``). To specify which hosts get a ``ceph.conf``:: -Then, run bootstrap referencing this file:: + ceph config set mgr mgr/cephadm/manage_etc_ceph_ceph_conf_hosts - cephadm bootstrap -c /root/ceph.conf ... +For example, to distribute configs to hosts with the ``bare_config`` label,:: + ceph config set mgr mgr/cephadm/manage_etc_ceph_ceph_conf_hosts label:bare_config +(See :ref:`orchestrator-cli-placement-spec` for more information about placement specs.) diff --git a/ceph/doc/cephadm/osd.rst b/ceph/doc/cephadm/osd.rst index e61ef1534..5c01d038f 100644 --- a/ceph/doc/cephadm/osd.rst +++ b/ceph/doc/cephadm/osd.rst @@ -171,15 +171,27 @@ For example: Declarative State ----------------- -Note that the effect of ``ceph orch apply`` is persistent; that is, drives which are added to the system -or become available (say, by zapping) after the command is complete will be automatically found and added to the cluster. +The effect of ``ceph orch apply`` is persistent. This means that drives that +are added to the system after the ``ceph orch apply`` command completes will be +automatically found and added to the cluster. It also means that drives that +become available (by zapping, for example) after the ``ceph orch apply`` +command completes will be automatically found and added to the cluster. -That is, after using:: +We will examine the effects of the following command: - ceph orch apply osd --all-available-devices + .. prompt:: bash # + + ceph orch apply osd --all-available-devices + +After running the above command: + +* If you add new disks to the cluster, they will automatically be used to + create new OSDs. +* If you remove an OSD and clean the LVM physical volume, a new OSD will be + created automatically. -* If you add new disks to the cluster they will automatically be used to create new OSDs. -* A new OSD will be created automatically if you remove an OSD and clean the LVM physical volume. +To disable the automatic creation of OSD on available devices, use the +``unmanaged`` parameter: If you want to avoid this behavior (disable automatic creation of OSD on available devices), use the ``unmanaged`` parameter: @@ -187,6 +199,16 @@ If you want to avoid this behavior (disable automatic creation of OSD on availab ceph orch apply osd --all-available-devices --unmanaged=true +.. note:: + + Keep these three facts in mind: + + - The default behavior of ``ceph orch apply`` causes cephadm constantly to reconcile. This means that cephadm creates OSDs as soon as new drives are detected. + + - Setting ``unmanaged: True`` disables the creation of OSDs. If ``unmanaged: True`` is set, nothing will happen even if you apply a new OSD service. + + - ``ceph orch daemon add`` creates OSDs, but does not add an OSD service. + * For cephadm, see also :ref:`cephadm-spec-unmanaged`. @@ -246,7 +268,7 @@ It is possible to stop queued OSD removals by using the following command: .. prompt:: bash # - ceph orch osd rm stop + ceph orch osd rm stop Example: @@ -266,7 +288,7 @@ Replacing an OSD .. prompt:: bash # - orch osd rm --replace [--force] + orch osd rm --replace [--force] Example: @@ -341,25 +363,75 @@ Example command: device. To disable this behavior, see :ref:`cephadm-osd-declarative`. +.. _osd_autotune: + +Automatically tuning OSD memory +=============================== + +OSD daemons will adjust their memory consumption based on the +``osd_memory_target`` config option (several gigabytes, by +default). If Ceph is deployed on dedicated nodes that are not sharing +memory with other services, cephadm can automatically adjust the per-OSD +memory consumption based on the total amount of RAM and the number of deployed +OSDs. + +This option is enabled globally with:: + + ceph config set osd osd_memory_target_autotune true + +Cephadm will start with a fraction +(``mgr/cephadm/autotune_memory_target_ratio``, which defaults to +``.7``) of the total RAM in the system, subtract off any memory +consumed by non-autotuned daemons (non-OSDs, for OSDs for which +``osd_memory_target_autotune`` is false), and then divide by the +remaining OSDs. + +The final targets are reflected in the config database with options like:: + + WHO MASK LEVEL OPTION VALUE + osd host:foo basic osd_memory_target 126092301926 + osd host:bar basic osd_memory_target 6442450944 + +Both the limits and the current memory consumed by each daemon are visible from +the ``ceph orch ps`` output in the ``MEM LIMIT`` column:: + + NAME HOST PORTS STATUS REFRESHED AGE MEM USED MEM LIMIT VERSION IMAGE ID CONTAINER ID + osd.1 dael running (3h) 10s ago 3h 72857k 117.4G 17.0.0-3781-gafaed750 7015fda3cd67 9e183363d39c + osd.2 dael running (81m) 10s ago 81m 63989k 117.4G 17.0.0-3781-gafaed750 7015fda3cd67 1f0cc479b051 + osd.3 dael running (62m) 10s ago 62m 64071k 117.4G 17.0.0-3781-gafaed750 7015fda3cd67 ac5537492f27 + +To exclude an OSD from memory autotuning, disable the autotune option +for that OSD and also set a specific memory target. For example, + + .. prompt:: bash # + + ceph config set osd.123 osd_memory_target_autotune false + ceph config set osd.123 osd_memory_target 16G + + .. _drivegroups: Advanced OSD Service Specifications =================================== -:ref:`orchestrator-cli-service-spec` of type ``osd`` are a way to describe a cluster layout using the properties of disks. -It gives the user an abstract way tell ceph which disks should turn into an OSD -with which configuration without knowing the specifics of device names and paths. +:ref:`orchestrator-cli-service-spec`\s of type ``osd`` are a way to describe a +cluster layout, using the properties of disks. Service specifications give the +user an abstract way to tell Ceph which disks should turn into OSDs with which +configurations, without knowing the specifics of device names and paths. + +Service specifications make it possible to define a yaml or json file that can +be used to reduce the amount of manual work involved in creating OSDs. -Instead of doing this +For example, instead of running the following command: .. prompt:: bash [monitor.1]# ceph orch daemon add osd **:** -for each device and each host, we can define a yaml|json file that allows us to describe -the layout. Here's the most basic example. +for each device and each host, we can define a yaml or json file that allows us +to describe the layout. Here's the most basic example. -Create a file called i.e. osd_spec.yml +Create a file called (for example) ``osd_spec.yml``: .. code-block:: yaml @@ -370,58 +442,60 @@ Create a file called i.e. osd_spec.yml data_devices: <- the type of devices you are applying specs to all: true <- a filter, check below for a full list -This would translate to: +This means : -Turn any available(ceph-volume decides what 'available' is) into an OSD on all hosts that match -the glob pattern '*'. (The glob pattern matches against the registered hosts from `host ls`) -There will be a more detailed section on host_pattern down below. +#. Turn any available device (ceph-volume decides what 'available' is) into an + OSD on all hosts that match the glob pattern '*'. (The glob pattern matches + against the registered hosts from `host ls`) A more detailed section on + host_pattern is available below. -and pass it to `osd create` like so +#. Then pass it to `osd create` like this: -.. prompt:: bash [monitor.1]# + .. prompt:: bash [monitor.1]# - ceph orch apply osd -i /path/to/osd_spec.yml + ceph orch apply osd -i /path/to/osd_spec.yml -This will go out on all the matching hosts and deploy these OSDs. + This instruction will be issued to all the matching hosts, and will deploy + these OSDs. -Since we want to have more complex setups, there are more filters than just the 'all' filter. + Setups more complex than the one specified by the ``all`` filter are + possible. See :ref:`osd_filters` for details. -Also, there is a `--dry-run` flag that can be passed to the `apply osd` command, which gives you a synopsis -of the proposed layout. + A ``--dry-run`` flag can be passed to the ``apply osd`` command to display a + synopsis of the proposed layout. Example .. prompt:: bash [monitor.1]# - [monitor.1]# ceph orch apply osd -i /path/to/osd_spec.yml --dry-run + ceph orch apply osd -i /path/to/osd_spec.yml --dry-run + +.. _osd_filters: Filters ------- .. note:: - Filters are applied using a `AND` gate by default. This essentially means that a drive needs to fulfill all filter - criteria in order to get selected. - If you wish to change this behavior you can adjust this behavior by setting - - `filter_logic: OR` # valid arguments are `AND`, `OR` - - in the OSD Specification. + Filters are applied using an `AND` gate by default. This means that a drive + must fulfill all filter criteria in order to get selected. This behavior can + be adjusted by setting ``filter_logic: OR`` in the OSD specification. -You can assign disks to certain groups by their attributes using filters. +Filters are used to assign disks to groups, using their attributes to group +them. -The attributes are based off of ceph-volume's disk query. You can retrieve the information -with +The attributes are based off of ceph-volume's disk query. You can retrieve +information about the attributes with this command: .. code-block:: bash ceph-volume inventory -Vendor or Model: -^^^^^^^^^^^^^^^^ +Vendor or Model +^^^^^^^^^^^^^^^ -You can target specific disks by their Vendor or by their Model +Specific disks can be targeted by vendor or model: .. code-block:: yaml @@ -434,19 +508,19 @@ or vendor: disk_vendor_name -Size: -^^^^^ +Size +^^^^ -You can also match by disk `Size`. +Specific disks can be targeted by `Size`: .. code-block:: yaml size: size_spec -Size specs: -___________ +Size specs +__________ -Size specification of format can be of form: +Size specifications can be of the following forms: * LOW:HIGH * :HIGH @@ -455,38 +529,38 @@ Size specification of format can be of form: Concrete examples: -Includes disks of an exact size +To include disks of an exact size .. code-block:: yaml size: '10G' -Includes disks which size is within the range +To include disks within a given range of size: .. code-block:: yaml size: '10G:40G' -Includes disks less than or equal to 10G in size +To include disks that are less than or equal to 10G in size: .. code-block:: yaml size: ':10G' - -Includes disks equal to or greater than 40G in size +To include disks equal to or greater than 40G in size: .. code-block:: yaml size: '40G:' -Sizes don't have to be exclusively in Gigabyte(G). +Sizes don't have to be specified exclusively in Gigabytes(G). -Supported units are Megabyte(M), Gigabyte(G) and Terrabyte(T). Also appending the (B) for byte is supported. MB, GB, TB +Other units of size are supported: Megabyte(M), Gigabyte(G) and Terrabyte(T). +Appending the (B) for byte is also supported: ``MB``, ``GB``, ``TB``. -Rotational: -^^^^^^^^^^^ +Rotational +^^^^^^^^^^ This operates on the 'rotational' attribute of the disk. @@ -499,8 +573,8 @@ This operates on the 'rotational' attribute of the disk. `0` to match all disks that are non-rotational (SSD, NVME etc) -All: -^^^^ +All +^^^ This will take all disks that are 'available' @@ -511,17 +585,17 @@ Note: This is exclusive for the data_devices section. all: true -Limiter: -^^^^^^^^ +Limiter +^^^^^^^ -When you specified valid filters but want to limit the amount of matching disks you can use the 'limit' directive. +If you have specified some valid filters but want to limit the number of disks that they match, use the ``limit`` directive: .. code-block:: yaml limit: 2 -For example, if you used `vendor` to match all disks that are from `VendorA` but only want to use the first two -you could use `limit`. +For example, if you used `vendor` to match all disks that are from `VendorA` +but want to use only the first two, you could use `limit`: .. code-block:: yaml @@ -529,7 +603,7 @@ you could use `limit`. vendor: VendorA limit: 2 -Note: Be aware that `limit` is really just a last resort and shouldn't be used if it can be avoided. +Note: `limit` is a last resort and shouldn't be used if it can be avoided. Additional Options diff --git a/ceph/doc/cephadm/rgw.rst b/ceph/doc/cephadm/rgw.rst index 1815776df..8329ee27b 100644 --- a/ceph/doc/cephadm/rgw.rst +++ b/ceph/doc/cephadm/rgw.rst @@ -112,18 +112,21 @@ elected as master, and the virtual IP will be moved to that node. The active haproxy acts like a load balancer, distributing all RGW requests between all the RGW daemons available. -**Prerequisites:** +Prerequisites +------------- * An existing RGW service, without SSL. (If you want SSL service, the certificate should be configured on the ingress service, not the RGW service.) -**Deploy of the high availability service for RGW** +Deploying +--------- Use the command:: ceph orch apply -i -**Service specification file:** +Service specification +--------------------- It is a yaml format file with the following properties: @@ -171,7 +174,10 @@ where the properties of this service specification are: SSL certificate, if SSL is to be enabled. This must contain the both the certificate and private key blocks in .pem format. -**Selecting ethernet interfaces for the virtual IP:** +.. _ingress-virtual-ip: + +Selecting ethernet interfaces for the virtual IP +------------------------------------------------ You cannot simply provide the name of the network interface on which to configure the virtual IP because interface names tend to vary @@ -204,7 +210,8 @@ configuring a "dummy" IP address is an unroutable network on the correct interfa and reference that dummy network in the networks list (see above). -**Useful hints for ingress:** +Useful hints for ingress +------------------------ -* Good to have at least 3 RGW daemons -* Use at least 3 hosts for the ingress +* It is good to have at least 3 RGW daemons. +* We recommend at least 3 hosts for the ingress service. diff --git a/ceph/doc/cephadm/service-management.rst b/ceph/doc/cephadm/service-management.rst index 26533544c..02424d492 100644 --- a/ceph/doc/cephadm/service-management.rst +++ b/ceph/doc/cephadm/service-management.rst @@ -5,39 +5,60 @@ Service Management Service Status ============== -A service is a group of daemons that are configured together. +A service is a group of daemons configured together. To see the status of one +of the services running in the Ceph cluster, do the following: -Print a list of services known to the orchestrator. The list can be limited to -services on a particular host with the optional --host parameter and/or -services of a particular type via optional --type parameter -(mon, osd, mgr, mds, rgw): +#. Use the command line to print a list of services. +#. Locate the service whose status you want to check. +#. Print the status of the service. -:: +The following command prints a list of services known to the orchestrator. To +limit the output to services only on a specified host, use the optional +``--host`` parameter. To limit the output to services of only a particular +type, use the optional ``--type`` parameter (mon, osd, mgr, mds, rgw): - ceph orch ls [--service_type type] [--service_name name] [--export] [--format f] [--refresh] + .. prompt:: bash # -Discover the status of a particular service or daemons:: + ceph orch ls [--service_type type] [--service_name name] [--export] [--format f] [--refresh] - ceph orch ls --service_type type --service_name [--refresh] +Discover the status of a particular service or daemon: -Export the service specs known to the orchestrator as yaml in format -that is compatible to ``ceph orch apply -i``:: + .. prompt:: bash # - ceph orch ls --export + ceph orch ls --service_type type --service_name [--refresh] -For examples about retrieving specs of single services see :ref:`orchestrator-cli-service-spec-retrieve`. +To export the service specifications knows to the orchestrator, run the following command. + + .. prompt:: bash # + + ceph orch ls --export + +The service specifications exported with this command will be exported as yaml +and that yaml can be used with the ``ceph orch apply -i`` command. + +For information about retrieving the specifications of single services (including examples of commands), see :ref:`orchestrator-cli-service-spec-retrieve`. Daemon Status ============= -A daemon is a running systemd unit and is part of a service. +A daemon is a systemd unit that is running and part of a service. -Print a list of all daemons known to the orchestrator:: +To see the status of a daemon, do the following: + +#. Print a list of all daemons known to the orchestrator. +#. Query the status of the target daemon. + +First, print a list of all daemons known to the orchestrator: + + .. prompt:: bash # ceph orch ps [--hostname host] [--daemon_type type] [--service_name name] [--daemon_id id] [--format f] [--refresh] -Query the status of a particular service instance (mon, osd, mds, rgw). For OSDs -the id is the numeric OSD ID, for MDS services it is the file system name:: +Then query the status of a particular service instance (mon, osd, mds, rgw). +For OSDs the id is the numeric OSD ID. For MDS services the id is the file +system name: + + .. prompt:: bash # ceph orch ps --daemon_type osd --daemon_id 0 @@ -46,8 +67,8 @@ the id is the numeric OSD ID, for MDS services it is the file system name:: Service Specification ===================== -A *Service Specification* is a data structure -to specify the deployment of services. For example in YAML: +A *Service Specification* is a data structure that is used to specify the +deployment of services. Here is an example of a service specification in YAML: .. code-block:: yaml @@ -61,7 +82,7 @@ to specify the deployment of services. For example in YAML: unmanaged: false ... -where the properties of a service specification are: +In this example, the properties of this service specification are: * ``service_type`` The type of the service. Needs to be either a Ceph @@ -73,21 +94,20 @@ where the properties of a service specification are: The name of the service. * ``placement`` See :ref:`orchestrator-cli-placement-spec`. -* ``unmanaged`` - If set to ``true``, the orchestrator will not deploy nor - remove any daemon associated with this service. Placement and all other - properties will be ignored. This is useful, if this service should not - be managed temporarily. For cephadm, See :ref:`cephadm-spec-unmanaged` +* ``unmanaged`` If set to ``true``, the orchestrator will not deploy nor remove + any daemon associated with this service. Placement and all other properties + will be ignored. This is useful, if you do not want this service to be + managed temporarily. For cephadm, See :ref:`cephadm-spec-unmanaged` -Each service type can have additional service specific properties. +Each service type can have additional service-specific properties. Service specifications of type ``mon``, ``mgr``, and the monitoring types do not require a ``service_id``. A service of type ``osd`` is described in :ref:`drivegroups` -Many service specifications can be applied at once using -``ceph orch apply -i`` by submitting a multi-document YAML file:: +Many service specifications can be applied at once using ``ceph orch apply -i`` +by submitting a multi-document YAML file:: cat <. --export > rgw...yaml ceph orch ls --service-type mgr --export > mgr.yaml @@ -132,10 +154,16 @@ For the orchestrator to deploy a *service*, it needs to know where to deploy specification. Placement specifications can either be passed as command line arguments or in a YAML files. +.. note:: + + cephadm will not deploy daemons on hosts with the ``_no_schedule`` label; see :ref:`cephadm-special-host-labels`. + Explicit placements ------------------- -Daemons can be explicitly placed on hosts by simply specifying them:: +Daemons can be explicitly placed on hosts by simply specifying them: + + .. prompt:: bash # orch apply prometheus --placement="host1 host2 host3" @@ -150,9 +178,11 @@ Or in YAML: - host2 - host3 -MONs and other services may require some enhanced network specifications:: +MONs and other services may require some enhanced network specifications: + + .. prompt:: bash # - orch daemon add mon --placement="myhost:[v2:1.2.3.4:3300,v1:1.2.3.4:6789]=name" + orch daemon add mon --placement="myhost:[v2:1.2.3.4:3300,v1:1.2.3.4:6789]=name" where ``[v2:1.2.3.4:3300,v1:1.2.3.4:6789]`` is the network address of the monitor and ``=name`` specifies the name of the new monitor. @@ -162,7 +192,9 @@ and ``=name`` specifies the name of the new monitor. Placement by labels ------------------- -Daemons can be explicitly placed on hosts that match a specific label:: +Daemons can be explicitly placed on hosts that match a specific label: + + .. prompt:: bash # orch apply prometheus --placement="label:mylabel" @@ -179,7 +211,9 @@ Or in YAML: Placement by pattern matching ----------------------------- -Daemons can be placed on hosts as well:: +Daemons can be placed on hosts as well: + + .. prompt:: bash # orch apply prometheus --placement='myhost[1-3]' @@ -191,7 +225,9 @@ Or in YAML: placement: host_pattern: "myhost[1-3]" -To place a service on *all* hosts, use ``"*"``:: +To place a service on *all* hosts, use ``"*"``: + + .. prompt:: bash # orch apply node-exporter --placement='*' @@ -207,21 +243,27 @@ Or in YAML: Setting a limit --------------- -By specifying ``count``, only that number of daemons will be created:: +By specifying ``count``, only the number of daemons specified will be created: + + .. prompt:: bash # orch apply prometheus --placement=3 -To deploy *daemons* on a subset of hosts, also specify the count:: +To deploy *daemons* on a subset of hosts, specify the count: + + .. prompt:: bash # orch apply prometheus --placement="2 host1 host2 host3" -If the count is bigger than the amount of hosts, cephadm deploys one per host:: +If the count is bigger than the amount of hosts, cephadm deploys one per host: + + .. prompt:: bash # orch apply prometheus --placement="3 host1 host2" -results in two Prometheus daemons. +The command immediately above results in two Prometheus daemons. -Or in YAML: +YAML can also be used to specify limits, in the following way: .. code-block:: yaml @@ -229,7 +271,7 @@ Or in YAML: placement: count: 3 -Or with hosts: +YAML can also be used to specify limits on hosts: .. code-block:: yaml @@ -249,15 +291,21 @@ service in a ``ServiceSpec``. For certain operations, like updating the RGW HTTP port, we need to update the existing specification. -1. List the current ``ServiceSpec``:: +1. List the current ``ServiceSpec``: + + .. prompt:: bash # ceph orch ls --service_name= --export > myservice.yaml -2. Update the yaml file:: +2. Update the yaml file: + + .. prompt:: bash # vi myservice.yaml -3. Apply the new ``ServiceSpec``:: +3. Apply the new ``ServiceSpec``: + + .. prompt:: bash # ceph orch apply -i myservice.yaml [--dry-run] @@ -268,22 +316,25 @@ Cephadm uses a declarative state to define the layout of the cluster. This state consists of a list of service specifications containing placement specifications (See :ref:`orchestrator-cli-service-spec` ). -Cephadm constantly compares list of actually running daemons in the cluster -with the desired service specifications and will either add or remove new -daemons. +Cephadm continually compares a list of daemons actually running in the cluster +against the list in the service specifications. Cephadm adds new daemons and +removes old daemons as necessary in order to conform to the service +specifications. + +Cephadm does the following to maintain compliance with the service +specifications. -First, cephadm will select a list of candidate hosts. It first looks for -explicit host names and will select those. In case there are no explicit hosts -defined, cephadm looks for a label specification. If there is no label defined -in the specification, cephadm will select hosts based on a host pattern. If -there is no pattern defined, cepham will finally select all known hosts as -candidates. +Cephadm first selects a list of candidate hosts. Cephadm seeks explicit host +names and selects them. If cephadm finds no explicit host names, it looks for +label specifications. If no label is defined in the specification, cephadm +selects hosts based on a host pattern. If no host pattern is defined, as a last +resort, cephadm selects all known hosts as candidates. -Then, cephadm will consider existing daemons of this services and will try to -avoid moving any daemons. +Cephadm is aware of existing daemons running services and tries to avoid moving +them. -Cephadm supports the deployment of a specific amount of services. Let's -consider a service specification like so: +Cephadm supports the deployment of a specific amount of services. +Consider the following service specification: .. code-block:: yaml @@ -293,34 +344,39 @@ consider a service specification like so: count: 3 label: myfs -This instructs cephadm to deploy three daemons on hosts labeled with -``myfs`` across the cluster. +This service specifcation instructs cephadm to deploy three daemons on hosts +labeled ``myfs`` across the cluster. -Then, in case there are less than three daemons deployed on the candidate -hosts, cephadm will then randomly choose hosts for deploying new daemons. +If there are fewer than three daemons deployed on the candidate hosts, cephadm +randomly chooses hosts on which to deploy new daemons. -In case there are more than three daemons deployed, cephadm will remove -existing daemons. +If there are more than three daemons deployed on the candidate hosts, cephadm +removes existing daemons. -Finally, cephadm will remove daemons on hosts that are outside of the list of +Finally, cephadm removes daemons on hosts that are outside of the list of candidate hosts. -However, there is a special cases that cephadm needs to consider. +.. note:: + + There is a special case that cephadm must consider. -In case the are fewer hosts selected by the placement specification than -demanded by ``count``, cephadm will only deploy on selected hosts. + If there are fewer hosts selected by the placement specification than + demanded by ``count``, cephadm will deploy only on the selected hosts. .. _cephadm-spec-unmanaged: -Disable automatic deployment of daemons -======================================= +Disabling automatic deployment of daemons +========================================= -Cephadm supports disabling the automated deployment and removal of daemons per service. In -this case, the CLI supports two commands that are dedicated to this mode. +Cephadm supports disabling the automated deployment and removal of daemons on a +per service basis. The CLI supports two commands for this. -To disable the automatic management of dameons, apply -the :ref:`orchestrator-cli-service-spec` with ``unmanaged=True``. +Disabling automatic management of daemons +----------------------------------------- + +To disable the automatic management of dameons, set ``unmanaged=True`` in the +:ref:`orchestrator-cli-service-spec` (``mgr.yaml``). ``mgr.yaml``: @@ -331,43 +387,55 @@ the :ref:`orchestrator-cli-service-spec` with ``unmanaged=True``. placement: label: mgr -.. code-block:: bash - ceph orch apply -i mgr.yaml +.. prompt:: bash # + + ceph orch apply -i mgr.yaml + .. note:: - cephadm will no longer deploy any new daemons, if the placement - specification matches additional hosts. + After you apply this change in the Service Specification, cephadm will no + longer deploy any new daemons (even if the placement specification matches + additional hosts). + +Deploying a daemon on a host manually +------------------------------------- -To manually deploy a daemon on a host, please execute: +To manually deploy a daemon on a host, run a command of the following form: -.. code-block:: bash + .. prompt:: bash # - ceph orch daemon add --placement= + ceph orch daemon add --placement= -For example +For example : -.. code-block:: bash + .. prompt:: bash # - ceph orch daemon add mgr --placement=my_host + ceph orch daemon add mgr --placement=my_host -To manually remove a daemon, please run: +Removing a daemon from a host manually +-------------------------------------- -.. code-block:: bash +To manually remove a daemon, run a command of the following form: - ceph orch daemon rm ... [--force] + .. prompt:: bash # -For example + ceph orch daemon rm ... [--force] -.. code-block:: bash +For example: - ceph orch daemon rm mgr.my_host.xyzxyz + .. prompt:: bash # + + ceph orch daemon rm mgr.my_host.xyzxyz .. note:: For managed services (``unmanaged=False``), cephadm will automatically deploy a new daemon a few seconds later. + +See also +-------- * See :ref:`cephadm-osd-declarative` for special handling of unmanaged OSDs. * See also :ref:`cephadm-pause` diff --git a/ceph/doc/cephadm/upgrade.rst b/ceph/doc/cephadm/upgrade.rst index 64ba6dadb..4e3e657bf 100644 --- a/ceph/doc/cephadm/upgrade.rst +++ b/ceph/doc/cephadm/upgrade.rst @@ -2,9 +2,9 @@ Upgrading Ceph ============== -Cephadm is capable of safely upgrading Ceph from one bugfix release to -another. For example, you can upgrade from v15.2.0 (the first Octopus -release) to the next point release v15.2.1. +Cephadm can safely upgrade Ceph from one bugfix release to the next. For +example, you can upgrade from v15.2.0 (the first Octopus release) to the next +point release, v15.2.1. The automated upgrade process follows Ceph best practices. For example: @@ -13,54 +13,72 @@ The automated upgrade process follows Ceph best practices. For example: will remain available. Keep in mind that the Ceph cluster health status is likely to switch to -`HEALTH_WARNING` during the upgrade. +``HEALTH_WARNING`` during the upgrade. Starting the upgrade ==================== -Before you start, you should verify that all hosts are currently online -and your cluster is healthy. +Before you begin using cephadm to upgrade Ceph, verify that all hosts are currently online and that your cluster is healthy: -:: +.. prompt:: bash # - # ceph -s + ceph -s + +To upgrade (or downgrade) to a specific release: -To upgrade (or downgrade) to a specific release:: +.. prompt:: bash # - # ceph orch upgrade start --ceph-version + ceph orch upgrade start --ceph-version -For example, to upgrade to v15.2.1:: +For example, to upgrade to v15.2.1: - # ceph orch upgrade start --ceph-version 15.2.1 +.. prompt:: bash # + + ceph orch upgrade start --ceph-version 15.2.1 Monitoring the upgrade ====================== -Determine whether an upgrade is in process and what version the cluster is -upgrading to with:: +Determine (1) whether an upgrade is in progress and (2) which version the +cluster is upgrading to by running the following command: + +.. prompt:: bash # - # ceph orch upgrade status + ceph orch upgrade status -While the upgrade is underway, you will see a progress bar in the ceph -status output. For example:: +Watching the progress bar during a Ceph upgrade +----------------------------------------------- + +During the upgrade, a progress bar is visible in the ceph status output. It +looks like this: + +.. code-block:: console # ceph -s + [...] progress: Upgrade to docker.io/ceph/ceph:v15.2.1 (00h 20m 12s) [=======.....................] (time remaining: 01h 43m 31s) -You can also watch the cephadm log with:: +Watching the cephadm log during an upgrade +------------------------------------------ + +Watch the cephadm log by running the following command: - # ceph -W cephadm +.. prompt:: bash # + + ceph -W cephadm Canceling an upgrade ==================== -You can stop the upgrade process at any time with:: +You can stop the upgrade process at any time with: + +.. prompt:: bash # # ceph orch upgrade stop @@ -73,46 +91,57 @@ There are a few health alerts that can arise during the upgrade process. UPGRADE_NO_STANDBY_MGR ---------------------- -Ceph requires an active and standby manager daemon in order to proceed, but -there is currently no standby. +This alert means that Ceph requires an active and standby manager daemon in +order to proceed, but there is currently no standby. + +You can ensure that Cephadm is configured to run 2 (or more) managers by running the following command: + +.. prompt:: bash # -You can ensure that Cephadm is configured to run 2 (or more) managers with:: + ceph orch apply mgr 2 # or more - # ceph orch apply mgr 2 # or more +You can check the status of existing mgr daemons by running the following command: -You can check the status of existing mgr daemons with:: +.. prompt:: bash # - # ceph orch ps --daemon-type mgr + ceph orch ps --daemon-type mgr -If an existing mgr daemon has stopped, you can try restarting it with:: +If an existing mgr daemon has stopped, you can try to restart it by running the following command: - # ceph orch daemon restart +.. prompt:: bash # + + ceph orch daemon restart UPGRADE_FAILED_PULL ------------------- -Ceph was unable to pull the container image for the target version. -This can happen if you specify an version or container image that does -not exist (e.g., 1.2.3), or if the container registry is not reachable from -one or more hosts in the cluster. +This alert means that Ceph was unable to pull the container image for the +target version. This can happen if you specify a version or container image +that does not exist (e.g. "1.2.3"), or if the container registry can not +be reached by one or more hosts in the cluster. -You can cancel the existing upgrade and specify a different target version with:: +To cancel the existing upgrade and to specify a different target version, run the following commands: - # ceph orch upgrade stop - # ceph orch upgrade start --ceph-version +.. prompt:: bash # + + ceph orch upgrade stop + ceph orch upgrade start --ceph-version Using customized container images ================================= -For most users, simplify specifying the Ceph version is sufficient. -Cephadm will locate the specific Ceph container image to use by -combining the ``container_image_base`` configuration option (default: -``docker.io/ceph/ceph``) with a tag of ``vX.Y.Z``. +For most users, upgrading requires nothing more complicated than specifying the +Ceph version number to upgrade to. In such cases, cephadm locates the specific +Ceph container image to use by combining the ``container_image_base`` +configuration option (default: ``docker.io/ceph/ceph``) with a tag of +``vX.Y.Z``. + +But it is possible to upgrade to an arbitrary container image, if that's what +you need. For example, the following command upgrades to a development build: -You can also upgrade to an arbitrary container image. For example, to -upgrade to a development build:: +.. prompt:: bash # - # ceph orch upgrade start --image quay.io/ceph-ci/ceph:recent-git-branch-name + ceph orch upgrade start --image quay.io/ceph-ci/ceph:recent-git-branch-name For more information about available container images, see :ref:`containers`. diff --git a/ceph/doc/cephfs/administration.rst b/ceph/doc/cephfs/administration.rst index 7fd83001b..c53161313 100644 --- a/ceph/doc/cephfs/administration.rst +++ b/ceph/doc/cephfs/administration.rst @@ -216,30 +216,15 @@ does not change a MDS; it manipulates the file system rank which has been marked damaged. -Minimum Client Version ----------------------- +Required Client Features +------------------------ -It is sometimes desirable to set the minimum version of Ceph that a client must be -running to connect to a CephFS cluster. Older clients may sometimes still be -running with bugs that can cause locking issues between clients (due to -capability release). CephFS provides a mechanism to set the minimum -client version: +It is sometimes desirable to set features that clients must support to talk to +CephFS. Clients without those features may disrupt other clients or behave in +surprising ways. Or, you may want to require newer features to prevent older +and possibly buggy clients from connecting. -:: - - fs set min_compat_client - -For example, to only allow Nautilus clients, use: - -:: - - fs set cephfs min_compat_client nautilus - -Clients running an older version will be automatically evicted. - -Enforcing minimum version of CephFS client is achieved by setting required -client features. Commands to manipulate required client features of a file -system: +Commands to manipulate required client features of a file system: :: @@ -252,8 +237,9 @@ To list all CephFS features fs feature ls +Clients that are missing newly added features will be evicted automatically. -CephFS features and first release they came out. +Here are the current CephFS features and first release they came out: +------------------+--------------+-----------------+ | Feature | Ceph release | Upstream Kernel | @@ -278,6 +264,8 @@ CephFS features and first release they came out. +------------------+--------------+-----------------+ | metric_collect | pacific | N/A | +------------------+--------------+-----------------+ +| alternate_name | pacific | PLANNED | ++------------------+--------------+-----------------+ CephFS Feature Descriptions @@ -328,6 +316,13 @@ delegated inode numbers is a prerequisite for client to do async file creation. Clients can send performance metric to MDS if MDS support this feature. +:: + + alternate_name + +Clients can set and understand "alternate names" for directory entries. This is +to be used for encrypted file name support. + Global settings --------------- diff --git a/ceph/doc/cephfs/cephfs-mirroring.rst b/ceph/doc/cephfs/cephfs-mirroring.rst new file mode 100644 index 000000000..e485ea3fe --- /dev/null +++ b/ceph/doc/cephfs/cephfs-mirroring.rst @@ -0,0 +1,385 @@ +.. _cephfs-mirroring: + +========================= +CephFS Snapshot Mirroring +========================= + +CephFS supports asynchronous replication of snapshots to a remote CephFS file system via +`cephfs-mirror` tool. Snapshots are synchronized by mirroring snapshot data followed by +creating a snapshot with the same name (for a given directory on the remote file system) as +the snapshot being synchronized. + +Requirements +------------ + +The primary (local) and secondary (remote) Ceph clusters version should be Pacific or later. + +Creating Users +-------------- + +Start by creating a user (on the primary/local cluster) for the mirror daemon. This user +requires write capability on the metadata pool to create RADOS objects (index objects) +for watch/notify operation and read capability on the data pool(s):: + + $ ceph auth get-or-create client.mirror mon 'profile cephfs-mirror' mds 'allow r' osd 'allow rw tag cephfs metadata=*, allow r tag cephfs data=*' mgr 'allow r' + +Create a user for each file system peer (on the secondary/remote cluster). This user needs +to have full capabilities on the MDS (to take snapshots) and the OSDs:: + + $ ceph fs authorize client.mirror_remote / rwps + +This user should be used (as part of peer specification) when adding a peer. + +Starting Mirror Daemon +---------------------- + +Mirror daemon should be spawned using `systemctl(1)` unit files:: + + $ systemctl enable cephfs-mirror@mirror + $ systemctl start cephfs-mirror@mirror + +`cephfs-mirror` daemon can be run in foreground using:: + + $ cephfs-mirror --id mirror --cluster site-a -f + +.. note:: User used here is `mirror` created in the `Creating Users` section. + +Interface +--------- + +`Mirroring` module (manager plugin) provides interfaces for managing directory snapshot +mirroring. Manager interfaces are (mostly) wrappers around monitor commands for managing +file system mirroring and is the recommended control interface. + +Mirroring Module +---------------- + +The mirroring module is responsible for assigning directories to mirror daemons for +synchronization. Multiple mirror daemons can be spawned to achieve concurrency in +directory snapshot synchronization. When mirror daemons are spawned (or terminated) +, the mirroring module discovers the modified set of mirror daemons and rebalances +the directory assignment amongst the new set thus providing high-availability. + +.. note:: Multiple mirror daemons is currently untested. Only a single mirror daemon + is recommended. + +Mirroring module is disabled by default. To enable mirroring use:: + + $ ceph mgr module enable mirroring + +Mirroring module provides a family of commands to control mirroring of directory +snapshots. To add or remove directories, mirroring needs to be enabled for a given +file system. To enable mirroring use:: + + $ ceph fs snapshot mirror enable + +.. note:: Mirroring module commands use `fs snapshot mirror` prefix as compared to + the monitor commands which `fs mirror` prefix. Make sure to use module + commands. + +To disable mirroring, use:: + + $ ceph fs snapshot mirror disable + +Once mirroring is enabled, add a peer to which directory snapshots are to be mirrored. +Peers follow `@` specification and get assigned a unique-id (UUID) +when added. See `Creating Users` section on how to create Ceph users for mirroring. + +To add a peer use:: + + $ ceph fs snapshot mirror peer_add [] [] [] + +`` is optional, and defaults to `` (on the remote cluster). + +This requires the remote cluster ceph configuration and user keyring to be available in +the primary cluster. See `Bootstrap Peers` section to avoid this. `peer_add` additionally +supports passing the remote cluster monitor address and the user key. However, bootstrapping +a peer is the recommended way to add a peer. + +.. note:: Only a single peer is supported right now. + +To remove a peer use:: + + $ ceph fs snapshot mirror peer_remove + +To list file system mirror peers use:: + + $ ceph fs snapshot mirror peer_list + +To configure a directory for mirroring, use:: + + $ ceph fs snapshot mirror add + +To stop a mirroring directory snapshots use:: + + $ ceph fs snapshot mirror remove + +Only absolute directory paths are allowed. Also, paths are normalized by the mirroring +module, therfore, `/a/b/../b` is equivalent to `/a/b`. + + $ mkdir -p /d0/d1/d2 + $ ceph fs snapshot mirror add cephfs /d0/d1/d2 + {} + $ ceph fs snapshot mirror add cephfs /d0/d1/../d1/d2 + Error EEXIST: directory /d0/d1/d2 is already tracked + +Once a directory is added for mirroring, its subdirectory or ancestor directories are +disallowed to be added for mirorring:: + + $ ceph fs snapshot mirror add cephfs /d0/d1 + Error EINVAL: /d0/d1 is a ancestor of tracked path /d0/d1/d2 + $ ceph fs snapshot mirror add cephfs /d0/d1/d2/d3 + Error EINVAL: /d0/d1/d2/d3 is a subtree of tracked path /d0/d1/d2 + +Commands to check directory mapping (to mirror daemons) and directory distribution are +detailed in `Mirroring Status` section. + +Bootstrap Peers +--------------- + +Adding a peer (via `peer_add`) requires the peer cluster configuration and user keyring +to be available in the primary cluster (manager host and hosts running the mirror daemon). +This can be avoided by bootstrapping and importing a peer token. Peer bootstrap involves +creating a bootstrap token on the peer cluster via:: + + $ ceph fs snapshot mirror peer_bootstrap create + +e.g.:: + + $ ceph fs snapshot mirror peer_bootstrap create backup_fs client.mirror_remote site-remote + {"token": "eyJmc2lkIjogIjBkZjE3MjE3LWRmY2QtNDAzMC05MDc5LTM2Nzk4NTVkNDJlZiIsICJmaWxlc3lzdGVtIjogImJhY2t1cF9mcyIsICJ1c2VyIjogImNsaWVudC5taXJyb3JfcGVlcl9ib290c3RyYXAiLCAic2l0ZV9uYW1lIjogInNpdGUtcmVtb3RlIiwgImtleSI6ICJBUUFhcDBCZ0xtRmpOeEFBVnNyZXozai9YYUV0T2UrbUJEZlJDZz09IiwgIm1vbl9ob3N0IjogIlt2MjoxOTIuMTY4LjAuNTo0MDkxOCx2MToxOTIuMTY4LjAuNTo0MDkxOV0ifQ=="} + +`site-name` refers to a user-defined string to identify the remote filesystem. In context +of `peer_add` interface, `site-name` is the passed in `cluster` name from `remote_cluster_spec`. + +Import the bootstrap token in the primary cluster via:: + + $ ceph fs snapshot mirror peer_bootstrap import + +e.g.:: + + $ ceph fs snapshot mirror peer_bootstrap import cephfs eyJmc2lkIjogIjBkZjE3MjE3LWRmY2QtNDAzMC05MDc5LTM2Nzk4NTVkNDJlZiIsICJmaWxlc3lzdGVtIjogImJhY2t1cF9mcyIsICJ1c2VyIjogImNsaWVudC5taXJyb3JfcGVlcl9ib290c3RyYXAiLCAic2l0ZV9uYW1lIjogInNpdGUtcmVtb3RlIiwgImtleSI6ICJBUUFhcDBCZ0xtRmpOeEFBVnNyZXozai9YYUV0T2UrbUJEZlJDZz09IiwgIm1vbl9ob3N0IjogIlt2MjoxOTIuMTY4LjAuNTo0MDkxOCx2MToxOTIuMTY4LjAuNTo0MDkxOV0ifQ== + +Mirroring Status +---------------- + +CephFS mirroring module provides `mirror daemon status` interface to check mirror daemon status:: + + $ ceph fs snapshot mirror daemon status + [ + { + "daemon_id": 284167, + "filesystems": [ + { + "filesystem_id": 1, + "name": "a", + "directory_count": 1, + "peers": [ + { + "uuid": "02117353-8cd1-44db-976b-eb20609aa160", + "remote": { + "client_name": "client.mirror_remote", + "cluster_name": "ceph", + "fs_name": "backup_fs" + }, + "stats": { + "failure_count": 1, + "recovery_count": 0 + } + } + ] + } + ] + } + ] + +An entry per mirror daemon instance is displayed along with information such as configured +peers and basic stats. For more detailed stats, use the admin socket interface as detailed +below. + +CephFS mirror daemons provide admin socket commands for querying mirror status. To check +available commands for mirror status use:: + + $ ceph --admin-daemon /path/to/mirror/daemon/admin/socket help + { + .... + .... + "fs mirror status cephfs@360": "get filesystem mirror status", + .... + .... + } + +Commands with `fs mirror status` prefix provide mirror status for mirror enabled +file systems. Note that `cephfs@360` is of format `filesystem-name@filesystem-id`. +This format is required since mirror daemons get asynchronously notified regarding +file system mirror status (A file system can be deleted and recreated with the same +name). + +Right now, the command provides minimal information regarding mirror status:: + + $ ceph --admin-daemon /var/run/ceph/cephfs-mirror.asok fs mirror status cephfs@360 + { + "rados_inst": "192.168.0.5:0/1476644347", + "peers": { + "a2dc7784-e7a1-4723-b103-03ee8d8768f8": { + "remote": { + "client_name": "client.mirror_remote", + "cluster_name": "site-a", + "fs_name": "backup_fs" + } + } + }, + "snap_dirs": { + "dir_count": 1 + } + } + +`Peers` section in the command output above shows the peer information such as unique +peer-id (UUID) and specification. The peer-id is required to remove an existing peer +as mentioned in the `Mirror Module and Interface` section. + +Command with `fs mirror peer status` prefix provide peer synchronization status. This +command is of format `filesystem-name@filesystem-id peer-uuid`:: + + $ ceph --admin-daemon /var/run/ceph/cephfs-mirror.asok fs mirror peer status cephfs@360 a2dc7784-e7a1-4723-b103-03ee8d8768f8 + { + "/d0": { + "state": "idle", + "last_synced_snap": { + "id": 120, + "name": "snap1", + "sync_duration": 0.079997898999999997, + "sync_time_stamp": "274900.558797s" + }, + "snaps_synced": 2, + "snaps_deleted": 0, + "snaps_renamed": 0 + } + } + +Synchronization stats such as `snaps_synced`, `snaps_deleted` and `snaps_renamed` are reset +on daemon restart and/or when a directory is reassigned to another mirror daemon (when +multiple mirror daemons are deployed). + +A directory can be in one of the following states:: + + - `idle`: The directory is currently not being synchronized + - `syncing`: The directory is currently being synchronized + - `failed`: The directory has hit upper limit of consecutive failures + +When a directory hits a configured number of consecutive synchronization failures, the +mirror daemon marks it as `failed`. Synchronization for these directories are retried. +By default, the number of consecutive failures before a directory is marked as failed +is controlled by `cephfs_mirror_max_consecutive_failures_per_directory` configuration +option (default: 10) and the retry interval for failed directories is controlled via +`cephfs_mirror_retry_failed_directories_interval` configuration option (default: 60s). + +E.g., adding a regular file for synchronization would result in failed status:: + + $ ceph fs snapshot mirror add cephfs /f0 + $ ceph --admin-daemon /var/run/ceph/cephfs-mirror.asok fs mirror peer status cephfs@360 a2dc7784-e7a1-4723-b103-03ee8d8768f8 + { + "/d0": { + "state": "idle", + "last_synced_snap": { + "id": 120, + "name": "snap1", + "sync_duration": 0.079997898999999997, + "sync_time_stamp": "274900.558797s" + }, + "snaps_synced": 2, + "snaps_deleted": 0, + "snaps_renamed": 0 + }, + "/f0": { + "state": "failed", + "snaps_synced": 0, + "snaps_deleted": 0, + "snaps_renamed": 0 + } + } + +This allows a user to add a non-existent directory for synchronization. The mirror daemon +would mark the directory as failed and retry (less frequently). When the directory comes +to existence, the mirror daemons would unmark the failed state upon successfull snapshot +synchronization. + +When mirroring is disabled, the respective `fs mirror status` command for the file system +will not show up in command help. + +Configuration Options +--------------------- + +``cephfs_mirror_max_concurrent_directory_syncs`` + +:Description: Maximum number of directory snapshots that can be synchronized concurrently by + cephfs-mirror daemon. Controls the number of synchronization threads. +:Type: 64-bit Integer Unsigned +:Default: ``3`` + +``cephfs_mirror_action_update_interval`` + +:Description: Interval in seconds to process pending mirror update actions. +:Type: Float +:Default: ``2`` + +``cephfs_mirror_restart_mirror_on_blocklist_interval`` + +:Description: Interval in seconds to restart blocklisted mirror instances. Setting to zero (0) + disables restarting blocklisted instances. +:Type: Float +:Default: ``30`` + +``cephfs_mirror_max_snapshot_sync_per_cycle`` + +:Description: Maximum number of snapshots to mirror when a directory is picked up for mirroring + by worker threads. +:Type: 64-bit Integer Unsigned +:Default: ``3`` + +``cephfs_mirror_directory_scan_interval`` + +:Description: Interval in seconds to scan configured directories for snapshot mirroring. +:Type: 64-bit Integer Unsigned +:Default: ``10`` + +``cephfs_mirror_max_consecutive_failures_per_directory`` + +:Description: Number of consecutive snapshot synchronization failues to mark a directory as + "failed". Failed directories are retried for synchronization less frequently. +:Type: 64-bit Integer Unsigned +:Default: ``10`` + +``cephfs_mirror_retry_failed_directories_interval`` + +:Description: Interval in seconds to retry synchronization for failed directories. +:Type: 64-bit Integer Unsigned +:Default: ``60`` + +``cephfs_mirror_restart_mirror_on_failure_interval`` + +:Description: Interval in seconds to restart failed mirror instances. Setting to zero (0) + disables restarting failed mirror instances. +:Type: Float +:Default: ``20`` + +``cephfs_mirror_mount_timeout`` + +:Description: Timeout in seconds for mounting primary or secondary (remote) ceph file system + by the cephfs-mirror daemon. Setting this to a higher value could result in the + mirror daemon getting stalled when mounting a file system if the cluster is not + reachable. This option is used to override the usual client_mount_timeout. +:Type: Float +:Default: ``10`` + + +Re-adding Peers +--------------- + +When re-adding (reassigning) a peer to a file system in another cluster, ensure that +all mirror daemons have stopped synchronization to the peer. This can be checked +via `fs mirror status` admin socket command (the `Peer UUID` should not show up +in the command output). Also, it is recommended to purge synchronized directories +from the peer before re-adding it to another file system (especially those directories +which might exist in the new primary file system). This is not required if re-adding +a peer to the same primary file system it was earlier synchronized from. diff --git a/ceph/doc/cephfs/fs-nfs-exports.rst b/ceph/doc/cephfs/fs-nfs-exports.rst index 65363e584..b11c88a19 100644 --- a/ceph/doc/cephfs/fs-nfs-exports.rst +++ b/ceph/doc/cephfs/fs-nfs-exports.rst @@ -1,3 +1,5 @@ +.. _cephfs-nfs: + ======================= CephFS Exports over NFS ======================= @@ -11,12 +13,14 @@ Requirements - ``nfs-ganesha``, ``nfs-ganesha-ceph``, ``nfs-ganesha-rados-grace`` and ``nfs-ganesha-rados-urls`` packages (version 3.3 and above) +.. note:: From Pacific, the nfs mgr module must be enabled prior to use. + Create NFS Ganesha Cluster ========================== .. code:: bash - $ ceph nfs cluster create [] + $ ceph nfs cluster create [] [--ingress --virtual-ip ] This creates a common recovery pool for all NFS Ganesha daemons, new user based on ``clusterid``, and a common NFS Ganesha config RADOS object. @@ -28,10 +32,6 @@ This creates a common recovery pool for all NFS Ganesha daemons, new user based Currently, NFS Ganesha daemon deployed by cephadm listens on the standard port. So only one daemon will be deployed on a host. -```` signifies the export type, which corresponds to the NFS Ganesha file -system abstraction layer (FSAL). Permissible values are ``"cephfs`` or -``rgw``, but currently only ``cephfs`` is supported. - ```` is an arbitrary string by which this NFS Ganesha cluster will be known. @@ -49,24 +49,20 @@ cluster):: "2 host1,host2" +To deploy NFS with an HA front-end (virtual IP and load balancer), add the +``--ingress`` flag and specify a virtual IP address. This will deploy a combination +of keepalived and haproxy to provide an high-availability NFS frontend for the NFS +service. + For more details, refer :ref:`orchestrator-cli-placement-spec` but keep in mind that specifying the placement via a YAML file is not supported. -Update NFS Ganesha Cluster -========================== - -.. code:: bash - - $ ceph nfs cluster update - -This updates the deployed cluster according to the placement value. - Delete NFS Ganesha Cluster ========================== .. code:: bash - $ ceph nfs cluster delete + $ ceph nfs cluster rm This deletes the deployed cluster. @@ -160,8 +156,8 @@ This removes the user defined configuration. Create CephFS Export ==================== -.. warning:: Currently, the volume/nfs interface is not integrated with dashboard. Both - dashboard and volume/nfs interface have different export requirements and +.. warning:: Currently, the nfs interface is not integrated with dashboard. Both + dashboard and nfs interface have different export requirements and create exports differently. Management of dashboard created exports is not supported. @@ -186,12 +182,14 @@ path is '/'. It need not be unique. Subvolume path can be fetched using: $ ceph fs subvolume getpath [--group_name ] +.. note:: Export creation is supported only for NFS Ganesha clusters deployed using nfs interface. + Delete CephFS Export ==================== .. code:: bash - $ ceph nfs export delete + $ ceph nfs export rm This deletes an export in an NFS Ganesha cluster, where: @@ -323,4 +321,23 @@ grace period. The exports can be mounted by .. note:: Only NFS v4.0+ is supported. +Troubleshooting +=============== + +Checking NFS-Ganesha logs with + +1) ``cephadm`` + + .. code:: bash + + $ cephadm logs --fsid --name nfs..hostname + +2) ``rook`` + + .. code:: bash + + $ kubectl logs -n rook-ceph rook-ceph-nfs-- nfs-ganesha + +Log level can be changed using `nfs cluster config set` command. + .. _NFS-Ganesha NFS Server: https://github.com/nfs-ganesha/nfs-ganesha/wiki diff --git a/ceph/doc/cephfs/index.rst b/ceph/doc/cephfs/index.rst index 94e5c0487..02ae3e9b2 100644 --- a/ceph/doc/cephfs/index.rst +++ b/ceph/doc/cephfs/index.rst @@ -92,7 +92,7 @@ Administration Upgrading old file systems CephFS Top Utility Scheduled Snapshots - + CephFS Snapshot Mirroring .. raw:: html diff --git a/ceph/doc/cephfs/multimds.rst b/ceph/doc/cephfs/multimds.rst index e22a84fac..dcbbfc51e 100644 --- a/ceph/doc/cephfs/multimds.rst +++ b/ceph/doc/cephfs/multimds.rst @@ -162,10 +162,12 @@ your metadata throughput with no other administrative intervention. Presently, there are two types of ephemeral pinning: -**Distributed Ephemeral Pins**: This policy indicates that **all** of a -directory's immediate children should be ephemerally pinned. The canonical -example would be the ``/home`` directory: we want every user's home directory -to be spread across the entire MDS cluster. This can be set via: +**Distributed Ephemeral Pins**: This policy causes a directory to fragment +(even well below the normal fragmentation thresholds) and distribute its +fragments as ephemerally pinned subtrees. This has the effect of distributing +immediate children across a range of MDS ranks. The canonical example use-case +would be the ``/home`` directory: we want every user's home directory to be +spread across the entire MDS cluster. This can be set via: :: @@ -219,18 +221,3 @@ For the reverse situation: The ``home/patrick`` directory and its children will be pinned to rank 2 because its export pin overrides the policy on ``home``. - -If a directory has an export pin and an ephemeral pin policy, the export pin -applies to the directory itself and the policy to its children. So: - -:: - - mkdir -p home/{patrick,john} - setfattr -n ceph.dir.pin -v 0 home - setfattr -n ceph.dir.pin.distributed -v 1 home - -The home directory inode (and all of its directory fragments) will always be -located on rank 0. All children including ``home/patrick`` and ``home/john`` -will be ephemerally pinned according to the distributed policy. This may only -matter for some obscure performance advantages. All the same, it's mentioned -here so the override policy is clear. diff --git a/ceph/doc/cephfs/snap-schedule.rst b/ceph/doc/cephfs/snap-schedule.rst index aa1f31ea9..6a9c0d176 100644 --- a/ceph/doc/cephfs/snap-schedule.rst +++ b/ceph/doc/cephfs/snap-schedule.rst @@ -89,9 +89,9 @@ path. Examples:: ceph fs snap-schedule status / - ceph fs snap-schedule status /foo/bar format=json + ceph fs snap-schedule status /foo/bar --format=json ceph fs snap-schedule list / - ceph fs snap-schedule list / recursive=true # list all schedules in the tree + ceph fs snap-schedule list / --recursive=true # list all schedules in the tree Add and remove schedules @@ -115,7 +115,7 @@ Examples:: ceph fs snap-schedule add / 1h 11:55 ceph fs snap-schedule add / 2h 11:55 ceph fs snap-schedule remove / 1h 11:55 # removes one single schedule - ceph fs snap-schedule remove / 1h # removes all schedules with repeat=1h + ceph fs snap-schedule remove / 1h # removes all schedules with --repeat=1h ceph fs snap-schedule remove / # removes all schedules on path / Add and remove retention policies diff --git a/ceph/doc/cephfs/upgrading.rst b/ceph/doc/cephfs/upgrading.rst index 3966079d3..e9df8c0c7 100644 --- a/ceph/doc/cephfs/upgrading.rst +++ b/ceph/doc/cephfs/upgrading.rst @@ -71,6 +71,12 @@ command. Older versions of Ceph require you to stop these daemons manually. ceph fs set max_mds +9. Restore setting for ``allow_standby_replay`` (if applicable): + +:: + + ceph fs set allow_standby_replay true + Upgrading pre-Firefly file systems past Jewel ============================================= diff --git a/ceph/doc/dev/cephadm/developing-cephadm.rst b/ceph/doc/dev/cephadm/developing-cephadm.rst index 6eb6cc1eb..9d6531d80 100644 --- a/ceph/doc/dev/cephadm/developing-cephadm.rst +++ b/ceph/doc/dev/cephadm/developing-cephadm.rst @@ -10,6 +10,7 @@ vstart --cephadm - Start a cluster with vstart, with cephadm configured - Manage any additional daemons with cephadm +- Requires compiled ceph binaries In this case, the mon and manager at a minimum are running in the usual vstart way, not managed by cephadm. But cephadm is enabled and the local @@ -87,6 +88,26 @@ When you're done, you can tear down the cluster with:: sudo ../src/ckill.sh # or, sudo ../src/cephadm/cephadm rm-cluster --force --fsid `cat fsid` +cephadm bootstrap --shared_ceph_folder +====================================== + +Cephadm can also be used directly without compiled ceph binaries. + +Run cephadm like so:: + + sudo ./cephadm bootstrap --mon-ip 127.0.0.1 \ + --ssh-private-key /home//.ssh/id_rsa \ + --skip-mon-network \ + --skip-monitoring-stack --single-host-defaults \ + --skip-dashboard \ + --shared_ceph_folder /home//path/to/ceph/ + +- ``~/.ssh/id_rsa`` is used as the cluster key. It is assumed that + this key is authorized to ssh with no passphrase to root@`hostname`. + +Source code changes made in the ``pybind/mgr/`` directory then +require a daemon restart to take effect. + Note regarding network calls from CLI handlers ============================================== @@ -102,3 +123,126 @@ another handler is executing. This means we should do very few synchronous calls to remote hosts. As a guideline, cephadm should do at most ``O(1)`` network calls in CLI handlers. Everything else should be done asynchronously in other threads, like ``serve()``. + +Kcli: a virtualization management tool to make easy orchestrators development +============================================================================= +`Kcli `_ is meant to interact with existing +virtualization providers (libvirt, KubeVirt, oVirt, OpenStack, VMware vSphere, +GCP and AWS) and to easily deploy and customize VMs from cloud images. + +It allows you to setup an environment with several vms with your preferred +configuration( memory, cpus, disks) and OS flavor. + +main advantages: +---------------- + - Is fast. Typically you can have a completely new Ceph cluster ready to debug + and develop orchestrator features in less than 5 minutes. + - Is a "near production" lab. The lab created with kcli is very near of "real" + clusters in QE labs or even in production. So easy to test "real things" in + almost "real environment" + - Is safe and isolated. Do not depend of the things you have installed in your + machine. And the vms are isolated from your environment. + - Easy to work "dev" environment. For "not compilated" software pieces, + for example any mgr module. It is an environment that allow you to test your + changes interactively. + +Installation: +------------- +Complete documentation in `kcli installation `_ +but we strongly suggest to use the container image approach. + +So things to do: + - 1. Review `requeriments `_ + and install/configure whatever you need to meet them. + - 2. get the kcli image and create one alias for executing the kcli command + :: + + # podman pull quay.io/karmab/kcli + # alias kcli='podman run --net host -it --rm --security-opt label=disable -v $HOME/.ssh:/root/.ssh -v $HOME/.kcli:/root/.kcli -v /var/lib/libvirt/images:/var/lib/libvirt/images -v /var/run/libvirt:/var/run/libvirt -v $PWD:/workdir -v /var/tmp:/ignitiondir quay.io/karmab/kcli' + +.. note:: /var/lib/libvirt/images can be customized.... be sure that you are + using this folder for your OS images + +.. note:: Once you have used your kcli tool to create and use different labs, we + suggest you to "save" and use your own kcli image. + Why?: kcli is alive and it changes (and for the moment only exists one tag ... + latest). Because we have more than enough with the current functionality, and + what we want is overall stability, + we suggest to store the kcli image you are using in a safe place and update + your kcli alias to use your own image. + +Test your kcli installation: +---------------------------- +See the kcli `basic usage workflow `_ + +Create a Ceph lab cluster +------------------------- +In order to make easy this task we are going to use a kcli plan. + +A kcli plan is a file where you can define the different settings you want to +have in a set of vms. +You can define hardware parameters (cpu, memory, disks ..), operating system and +it also allows you to automate the installation and configuration of any +software you want to have. + +There is a `repository `_ with a collection of +plans that can be used for different purposes. And we have predefined plans to +install Ceph clusters using Ceph ansible or cephadm, lets create our first Ceph +cluster using cephadm:: + +# kcli2 create plan -u https://github.com/karmab/kcli-plans/blob/master/ceph/ceph_cluster.yml + +This will create a set of three vms using the plan file pointed by the url. +After a few minutes (depend of your laptop power), lets examine the cluster: + +* Take a look to the vms created:: + + # kcli list vms + +* Enter in the bootstrap node:: + + # kcli ssh ceph-node-00 + +* Take a look to the ceph cluster installed:: + + [centos@ceph-node-00 ~]$ sudo -i + [root@ceph-node-00 ~]# cephadm version + [root@ceph-node-00 ~]# cephadm shell + [ceph: root@ceph-node-00 /]# ceph orch host ls + +Create a Ceph cluster to make easy developing in mgr modules (Orchestrators and Dashboard) +------------------------------------------------------------------------------------------ +The cephadm kcli plan (and cephadm) are prepared to do that. + +The idea behind this method is to replace several python mgr folders in each of +the ceph daemons with the source code folders in your host machine. +This "trick" will allow you to make changes in any orchestrator or dashboard +module and test them intermediately. (only needed to disable/enable the mgr module) + +So in order to create a ceph cluster for development purposes you must use the +same cephadm plan but with a new parameter pointing your Ceph source code folder:: + + # kcli create plan -u https://github.com/karmab/kcli-plans/blob/master/ceph/ceph_cluster.yml -P ceph_dev_folder=/home/mycodefolder/ceph + +Ceph Dashboard development +-------------------------- +Ceph dashboard module is not going to be loaded if previously you have not +generated the frontend bundle. + +For now, in order load properly the Ceph Dashboardmodule and to apply frontend +changes you have to run "ng build" on your laptop:: + + # Start local frontend build with watcher (in background): + sudo dnf install -y nodejs + cd + cd src/pybind/mgr/dashboard/frontend + sudo chown -R :root dist node_modules + NG_CLI_ANALYTICS=false npm ci + npm run build -- --deleteOutputPath=false --watch & + +After saving your changes, the frontend bundle will be built again. +When completed, you'll see:: + + "Localized bundle generation complete." + +Then you can reload your Dashboard browser tab. diff --git a/ceph/doc/dev/cephfs-mirroring.rst b/ceph/doc/dev/cephfs-mirroring.rst index ac2f13ef3..7ce6874db 100644 --- a/ceph/doc/dev/cephfs-mirroring.rst +++ b/ceph/doc/dev/cephfs-mirroring.rst @@ -115,7 +115,7 @@ Mirroring module provides a family of commands to control mirroring of directory snapshots. To add or remove directories, mirroring needs to be enabled for a given file system. To enable mirroring use:: - $ ceph fs snapshot mirror enable + $ ceph fs snapshot mirror enable .. note:: Mirroring module commands use `fs snapshot mirror` prefix as compared to the monitor commands which `fs mirror` prefix. Make sure to use module @@ -123,7 +123,7 @@ file system. To enable mirroring use:: To disable mirroring, use:: - $ ceph fs snapshot mirror disable + $ ceph fs snapshot mirror disable Once mirroring is enabled, add a peer to which directory snapshots are to be mirrored. Peers follow `@` specification and get assigned a unique-id (UUID) @@ -131,9 +131,9 @@ when added. See `Creating Users` section on how to create Ceph users for mirrori To add a peer use:: - $ ceph fs snapshot mirror peer_add [] [] [] + $ ceph fs snapshot mirror peer_add [] [] [] -`` is optional, and default to `` (on the remote cluster). +`` is optional, and default to `` (on the remote cluster). This requires the remote cluster ceph configuration and user keyring to be available in the primary cluster. See `Bootstrap Peers` section to avoid this. `peer_add` additionally @@ -144,21 +144,21 @@ a peer is the recommended way to add a peer. To remove a peer use:: - $ ceph fs snapshot mirror peer_remove + $ ceph fs snapshot mirror peer_remove .. note:: See `Mirror Daemon Status` section on how to figure out Peer UUID. To list file system mirror peers use:: - $ ceph fs snapshot mirror peer_list + $ ceph fs snapshot mirror peer_list To configure a directory for mirroring, use:: - $ ceph fs snapshot mirror add + $ ceph fs snapshot mirror add To stop a mirroring directory snapshots use:: - $ ceph fs snapshot mirror remove + $ ceph fs snapshot mirror remove Only absolute directory paths are allowed. Also, paths are normalized by the mirroring module, therfore, `/a/b/../b` is equivalent to `/a/b`. @@ -210,8 +210,49 @@ Mirror Daemon Status -------------------- Mirror daemons get asynchronously notified about changes in file system mirroring status -and/or peer updates. CephFS mirror daemons provide admin socket commands for querying -mirror status. To check available commands for mirror status use:: +and/or peer updates. + +CephFS mirroring module provides `mirror daemon status` interface to check mirror daemon +status:: + + $ ceph fs snapshot mirror daemon status + +E.g:: + + $ ceph fs snapshot mirror daemon status a | jq + [ + { + "daemon_id": 284167, + "filesystems": [ + { + "filesystem_id": 1, + "name": "a", + "directory_count": 1, + "peers": [ + { + "uuid": "02117353-8cd1-44db-976b-eb20609aa160", + "remote": { + "client_name": "client.mirror_remote", + "cluster_name": "ceph", + "fs_name": "backup_fs" + }, + "stats": { + "failure_count": 1, + "recovery_count": 0 + } + } + ] + } + ] + } + ] + +An entry per mirror daemon instance is displayed along with information such as configured +peers and basic stats. For more detailed stats, use the admin socket interface as detailed +below. + +CephFS mirror daemons provide admin socket commands for querying mirror status. To check +available commands for mirror status use:: $ ceph --admin-daemon /path/to/mirror/daemon/admin/socket help { diff --git a/ceph/doc/dev/developer_guide/dash-devel.rst b/ceph/doc/dev/developer_guide/dash-devel.rst index 130a6b9a6..6a3e7976b 100644 --- a/ceph/doc/dev/developer_guide/dash-devel.rst +++ b/ceph/doc/dev/developer_guide/dash-devel.rst @@ -423,6 +423,26 @@ Note: When using docker, as your device, you might need to run the script with sudo permissions. +run-cephadm-e2e-tests.sh +......................... + +``run-cephadm-e2e-tests.sh`` runs a subset of E2E tests to verify that the Dashboard and cephadm as +Orchestrator backend behave correctly. + +Prerequisites: you need to install `KCLI +`_ in your local machine. + +Note: + This script is aimed to be run as jenkins job so the cleanup is triggered only in a jenkins + environment. In local, the user will shutdown the cluster when desired (i.e. after debugging). + +Start E2E tests by running:: + + $ cd + $ sudo chown -R $(id -un) src/pybind/mgr/dashboard/frontend/dist src/pybind/mgr/dashboard/frontend/node_modules + $ ./src/pybind/mgr/dashboard/ci/cephadm/run-cephadm-e2e-tests.sh + $ kcli delete plan -y ceph # After tests finish. + Other running options ..................... @@ -1466,6 +1486,25 @@ same applies to other request types: | DELETE | Yes | delete | 204 | +--------------+------------+----------------+-------------+ +To use a custom endpoint for the above listed methods, you can +use ``@RESTController.MethodMap`` + +.. code-block:: python + + import cherrypy + from ..tools import ApiController, RESTController + + @RESTController.MethodMap(version='0.1') + def create(self): + return {"msg": "Hello"} + +This decorator supports three parameters to customize the +endpoint: + +* ``resource"``: resource id. +* ``status=200``: set the HTTP status response code +* ``version``: version + How to use a custom API endpoint in a RESTController? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1494,7 +1533,7 @@ used. To use a custom endpoint inside a restricted ``RESTController`` use def some_post_endpoint(self, **data): return {"msg": data} -Both decorators also support four parameters to customize the +Both decorators also support five parameters to customize the endpoint: * ``method="GET"``: the HTTP method allowed to access this endpoint. @@ -1503,6 +1542,7 @@ endpoint: * ``status=200``: set the HTTP status response code * ``query_params=[]``: list of method parameter names that correspond to URL query parameters. +* ``version``: version How to restrict access to a controller? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/ceph/doc/install/get-packages.rst b/ceph/doc/install/get-packages.rst index 40468eec8..41130e4e4 100644 --- a/ceph/doc/install/get-packages.rst +++ b/ceph/doc/install/get-packages.rst @@ -25,10 +25,13 @@ There are three ways to get packages: Install packages with cephadm ============================= -#. Download the cephadm script:: +#. Download the cephadm script - curl --silent --remote-name --location https://github.com/ceph/ceph/raw/octopus/src/cephadm/cephadm - chmod +x cephadm +.. prompt:: bash $ + :substitutions: + + curl --silent --remote-name --location https://github.com/ceph/ceph/raw/|stable-release|/src/cephadm/cephadm + chmod +x cephadm #. Configure the Ceph repository based on the release name:: diff --git a/ceph/doc/man/8/cephfs-top.rst b/ceph/doc/man/8/cephfs-top.rst index 2ff8f7c5c..bad687f9a 100644 --- a/ceph/doc/man/8/cephfs-top.rst +++ b/ceph/doc/man/8/cephfs-top.rst @@ -36,6 +36,42 @@ Options Perform a selftest. This mode performs a sanity check of ``stats`` module. +Descriptions of fields +====================== + +.. describe:: chit + + cap hit rate + +.. describe:: rlat + + read latency + +.. describe:: wlat + + write latency + +.. describe:: mlat + + metadata latency + +.. describe:: dlease + + dentry lease rate + +.. describe:: ofiles + + number of opened files + +.. describe:: oicaps + + number of pinned caps + +.. describe:: oinodes + + number of opened inodes + + Availability ============ diff --git a/ceph/doc/mgr/mds_autoscaler.rst b/ceph/doc/mgr/mds_autoscaler.rst index 39dadd882..46fc44155 100644 --- a/ceph/doc/mgr/mds_autoscaler.rst +++ b/ceph/doc/mgr/mds_autoscaler.rst @@ -1,31 +1,23 @@ MDS Autoscaler Module ===================== -The MDS Autoscaler Module monitors ``fsmap`` update notifications from the mgr -daemon and takes action to spawn or kill MDS daemons for a file-system as per -changes to the: +The MDS Autoscaler Module monitors file systems to ensure sufficient MDS +daemons are available. It works by adjusting the placement specification for +the orchestrator backend of the MDS service. To enable, use: -- ``max_mds`` config value -- ``standby_count_wanted`` config value -- standby promotions to active MDS state in case of active MDS rank death +.. sh: -Bumping up the ``max_mds`` config option value causes a standby mds to be promoted -to hold an active rank. This leads to a drop in standby mds count. The MDS -Autoscaler module detects this deficit and the orchestrator module is notified -about the required MDS count. The orchestrator back-end then takes necessary -measures to spawn standby MDSs. + ceph mgr module enable mds_autoscaler -Dropping the ``max_mds`` config option causes the orchestrator back-end to kill -standby mds to achieve the new reduced count. Preferably standby mds are chosen -to be killed when the ``max_mds`` count is dropped. +The module will monitor the following file system settings to inform +placement count adjustments: -An increment and decrement of the ``standby_count_wanted`` config option value -has a similar effect on the total MDS count. The orchestrator is notified about -the change and necessary action to spawn or kill standby MDSs is taken. +- ``max_mds`` file system setting +- ``standby_count_wanted`` file system setting -A death of an active MDS rank also causes promotion of a standby mds to occupy -the required active rank. The MDS Autoscaler notices the change in the standby -mds count and a message is passed to the orchestrator to maintain the necessary -MDS count. +The Ceph monitor daemons are still responsible for promoting or stopping MDS +according to these settings. The ``mds_autoscaler`` simply adjusts the +number of MDS which are spawned by the orchestrator. -NOTE: There is no CLI associated with the MDS Autoscaler Module. +.. note: There is no CLI or module configurations as of now. Enable or disable + the module to turn on or off. diff --git a/ceph/doc/rados/configuration/index.rst b/ceph/doc/rados/configuration/index.rst index 096530728..327259374 100644 --- a/ceph/doc/rados/configuration/index.rst +++ b/ceph/doc/rados/configuration/index.rst @@ -39,6 +39,7 @@ To optimize the performance of your cluster, refer to the following: mon-lookup-dns Heartbeat Settings OSD Settings + DmClock Settings BlueStore Settings FileStore Settings Journal Settings diff --git a/ceph/doc/rados/configuration/mclock-config-ref.rst b/ceph/doc/rados/configuration/mclock-config-ref.rst new file mode 100644 index 000000000..0f773698f --- /dev/null +++ b/ceph/doc/rados/configuration/mclock-config-ref.rst @@ -0,0 +1,368 @@ +======================== + mClock Config Reference +======================== + +.. index:: mclock; configuration + +Mclock profiles mask the low level details from users, making it +easier for them to configure mclock. + +To use mclock, you must provide the following input parameters: + +* total capacity of each OSD + +* an mclock profile to enable + +Using the settings in the specified profile, the OSD determines and applies the +lower-level mclock and Ceph parameters. The parameters applied by the mclock +profile make it possible to tune the QoS between client I/O, recovery/backfill +operations, and other background operations (for example, scrub, snap trim, and +PG deletion). These background activities are considered best-effort internal +clients of Ceph. + + +.. index:: mclock; profile definition + +mClock Profiles - Definition and Purpose +======================================== + +A mclock profile is *“a configuration setting that when applied on a running +Ceph cluster enables the throttling of the operations(IOPS) belonging to +different client classes (background recovery, scrub, snaptrim, client op, +osd subop)”*. + +The mclock profile uses the capacity limits and the mclock profile selected by +the user to determine the low-level mclock resource control parameters. + +Depending on the profile, lower-level mclock resource-control parameters and +some Ceph-configuration parameters are transparently applied. + +The low-level mclock resource control parameters are the *reservation*, +*limit*, and *weight* that provide control of the resource shares, as +described in the :ref:`dmclock-qos` section. + + +.. index:: mclock; profile types + +mClock Profile Types +==================== + +mclock profiles can be broadly classified into two types, + +- **Built-in**: Users can choose between the following built-in profile types: + + - **high_client_ops** (*default*): + This profile allocates more reservation and limit to external-client ops + as compared to background recoveries and other internal clients within + Ceph. This profile is enabled by default. + - **high_recovery_ops**: + This profile allocates more reservation to background recoveries as + compared to external clients and other internal clients within Ceph. For + example, an admin may enable this profile temporarily to speed-up background + recoveries during non-peak hours. + - **balanced**: + This profile allocates equal reservation to client ops and background + recovery ops. + +- **Custom**: This profile gives users complete control over all the mclock + configuration parameters. Using this profile is not recommended without + a deep understanding of mclock and related Ceph-configuration options. + +.. note:: Across the built-in profiles, internal clients of mclock (for example + "scrub", "snap trim", and "pg deletion") are given slightly lower + reservations, but higher weight and no limit. This ensures that + these operations are able to complete quickly if there are no other + competing services. + + +.. index:: mclock; built-in profiles + +mClock Built-in Profiles +======================== + +When a built-in profile is enabled, the mClock scheduler calculates the low +level mclock parameters [*reservation*, *weight*, *limit*] based on the profile +enabled for each client type. The mclock parameters are calculated based on +the max OSD capacity provided beforehand. As a result, the following mclock +config parameters cannot be modified when using any of the built-in profiles: + +- ``osd_mclock_scheduler_client_res`` +- ``osd_mclock_scheduler_client_wgt`` +- ``osd_mclock_scheduler_client_lim`` +- ``osd_mclock_scheduler_background_recovery_res`` +- ``osd_mclock_scheduler_background_recovery_wgt`` +- ``osd_mclock_scheduler_background_recovery_lim`` +- ``osd_mclock_scheduler_background_best_effort_res`` +- ``osd_mclock_scheduler_background_best_effort_wgt`` +- ``osd_mclock_scheduler_background_best_effort_lim`` + +The following Ceph options will not be modifiable by the user: + +- ``osd_max_backfills`` +- ``osd_recovery_max_active`` + +This is because the above options are internally modified by the mclock +scheduler in order to maximize the impact of the set profile. + +By default, the *high_client_ops* profile is enabled to ensure that a larger +chunk of the bandwidth allocation goes to client ops. Background recovery ops +are given lower allocation (and therefore take a longer time to complete). But +there might be instances that necessitate giving higher allocations to either +client ops or recovery ops. In order to deal with such a situation, you can +enable one of the alternate built-in profiles mentioned above. + +If any mClock profile (including "custom") is active, the following Ceph config +sleep options will be disabled, + +- ``osd_recovery_sleep`` +- ``osd_recovery_sleep_hdd`` +- ``osd_recovery_sleep_ssd`` +- ``osd_recovery_sleep_hybrid`` +- ``osd_scrub_sleep`` +- ``osd_delete_sleep`` +- ``osd_delete_sleep_hdd`` +- ``osd_delete_sleep_ssd`` +- ``osd_delete_sleep_hybrid`` +- ``osd_snap_trim_sleep`` +- ``osd_snap_trim_sleep_hdd`` +- ``osd_snap_trim_sleep_ssd`` +- ``osd_snap_trim_sleep_hybrid`` + +The above sleep options are disabled to ensure that mclock scheduler is able to +determine when to pick the next op from its operation queue and transfer it to +the operation sequencer. This results in the desired QoS being provided across +all its clients. + + +.. index:: mclock; enable built-in profile + +Steps to Enable mClock Profile +============================== + +The following sections outline the steps required to enable a mclock profile. + +Determining OSD Capacity Using Benchmark Tests +---------------------------------------------- + +To allow mclock to fulfill its QoS goals across its clients, it is most +important to have a good understanding of each OSD's capacity in terms of its +baseline throughputs (IOPS) across the Ceph nodes. To determine this capacity, +you must perform appropriate benchmarking tests. The steps for performing these +benchmarking tests are broadly outlined below. + +Any existing benchmarking tool can be used for this purpose. The following +steps use the *Ceph Benchmarking Tool* (cbt_). Regardless of the tool +used, the steps described below remain the same. + +As already described in the :ref:`dmclock-qos` section, the number of +shards and the bluestore's throttle parameters have an impact on the mclock op +queues. Therefore, it is critical to set these values carefully in order to +maximize the impact of the mclock scheduler. + +:Number of Operational Shards: + We recommend using the default number of shards as defined by the + configuration options ``osd_op_num_shards``, ``osd_op_num_shards_hdd``, and + ``osd_op_num_shards_ssd``. In general, a lower number of shards will increase + the impact of the mclock queues. + +:Bluestore Throttle Parameters: + We recommend using the default values as defined by + ``bluestore_throttle_bytes`` and ``bluestore_throttle_deferred_bytes``. But + these parameters may also be determined during the benchmarking phase as + described below. + +Benchmarking Test Steps Using CBT +````````````````````````````````` + +The steps below use the default shards and detail the steps used to determine the +correct bluestore throttle values. + +.. note:: These steps, although manual in April 2021, will be automated in the future. + +1. On the Ceph node hosting the OSDs, download cbt_ from git. +2. Install cbt and all the dependencies mentioned on the cbt github page. +3. Construct the Ceph configuration file and the cbt yaml file. +4. Ensure that the bluestore throttle options ( i.e. + ``bluestore_throttle_bytes`` and ``bluestore_throttle_deferred_bytes``) are + set to the default values. +5. Ensure that the test is performed on similar device types to get reliable + OSD capacity data. +6. The OSDs can be grouped together with the desired replication factor for the + test to ensure reliability of OSD capacity data. +7. After ensuring that the OSDs nodes are in the desired configuration, run a + simple 4KiB random write workload on the OSD(s) for 300 secs. +8. Note the overall throughput(IOPS) obtained from the cbt output file. This + value is the baseline throughput(IOPS) when the default bluestore + throttle options are in effect. +9. If the intent is to determine the bluestore throttle values for your + environment, then set the two options, ``bluestore_throttle_bytes`` and + ``bluestore_throttle_deferred_bytes`` to 32 KiB(32768 Bytes) each to begin + with. Otherwise, you may skip to the next section. +10. Run the 4KiB random write workload as before on the OSD(s) for 300 secs. +11. Note the overall throughput from the cbt log files and compare the value + against the baseline throughput in step 8. +12. If the throughput doesn't match with the baseline, increment the bluestore + throttle options by 2x and repeat steps 9 through 11 until the obtained + throughput is very close to the baseline value. + +For example, during benchmarking on a machine with NVMe SSDs, a value of 256 KiB for +both bluestore throttle and deferred bytes was determined to maximize the impact +of mclock. For HDDs, the corresponding value was 40 MiB, where the overall +throughput was roughly equal to the baseline throughput. Note that in general +for HDDs, the bluestore throttle values are expected to be higher when compared +to SSDs. + +.. _cbt: https://github.com/ceph/cbt + + +Specifying Max OSD Capacity +---------------------------- + +The steps in this section may be performed only if the max osd capacity is +different from the default values (SSDs: 21500 IOPS and HDDs: 315 IOPS). The +option ``osd_mclock_max_capacity_iops_[hdd, ssd]`` can be set by specifying it +in either the **[global]** section or in a specific OSD section (**[osd.x]** of +your Ceph configuration file). + +Alternatively, commands of the following form may be used: + + .. prompt:: bash # + + ceph config set [global, osd] osd_mclock_max_capacity_iops_[hdd,ssd] + +For example, the following command sets the max capacity for all the OSDs in a +Ceph node whose underlying device type is SSDs: + + .. prompt:: bash # + + ceph config set osd osd_mclock_max_capacity_iops_ssd 25000 + +To set the capacity for a specific OSD (for example "osd.0") whose underlying +device type is HDD, use a command like this: + + .. prompt:: bash # + + ceph config set osd.0 osd_mclock_max_capacity_iops_hdd 350 + + +Specifying Which mClock Profile to Enable +----------------------------------------- + +As already mentioned, the default mclock profile is set to *high_client_ops*. +The other values for the built-in profiles include *balanced* and +*high_recovery_ops*. + +If there is a requirement to change the default profile, then the option +``osd_mclock_profile`` may be set in the **[global]** or **[osd]** section of +your Ceph configuration file before bringing up your cluster. + +Alternatively, to change the profile during runtime, use the following command: + + .. prompt:: bash # + + ceph config set [global,osd] osd_mclock_profile + +For example, to change the profile to allow faster recoveries, the following +command can be used to switch to the *high_recovery_ops* profile: + + .. prompt:: bash # + + ceph config set osd osd_mclock_profile high_recovery_ops + +.. note:: The *custom* profile is not recommended unless you are an advanced user. + +And that's it! You are ready to run workloads on the cluster and check if the +QoS requirements are being met. + + +.. index:: mclock; config settings + +mClock Config Options +===================== + +``osd_mclock_profile`` + +:Description: This sets the type of mclock profile to use for providing QoS + based on operations belonging to different classes (background + recovery, scrub, snaptrim, client op, osd subop). Once a built-in + profile is enabled, the lower level mclock resource control + parameters [*reservation, weight, limit*] and some Ceph + configuration parameters are set transparently. Note that the + above does not apply for the *custom* profile. + +:Type: String +:Valid Choices: high_client_ops, high_recovery_ops, balanced, custom +:Default: ``high_client_ops`` + +``osd_mclock_max_capacity_iops`` + +:Description: Max IOPS capacity (at 4KiB block size) to consider per OSD + (overrides _ssd and _hdd if non-zero) + +:Type: Float +:Default: ``0.0`` + +``osd_mclock_max_capacity_iops_hdd`` + +:Description: Max IOPS capacity (at 4KiB block size) to consider per OSD (for + rotational media) + +:Type: Float +:Default: ``315.0`` + +``osd_mclock_max_capacity_iops_ssd`` + +:Description: Max IOPS capacity (at 4KiB block size) to consider per OSD (for + solid state media) + +:Type: Float +:Default: ``21500.0`` + +``osd_mclock_cost_per_io_usec`` + +:Description: Cost per IO in microseconds to consider per OSD (overrides _ssd + and _hdd if non-zero) + +:Type: Float +:Default: ``0.0`` + +``osd_mclock_cost_per_io_usec_hdd`` + +:Description: Cost per IO in microseconds to consider per OSD (for rotational + media) + +:Type: Float +:Default: ``25000.0`` + +``osd_mclock_cost_per_io_usec_ssd`` + +:Description: Cost per IO in microseconds to consider per OSD (for solid state + media) + +:Type: Float +:Default: ``50.0`` + +``osd_mclock_cost_per_byte_usec`` + +:Description: Cost per byte in microseconds to consider per OSD (overrides _ssd + and _hdd if non-zero) + +:Type: Float +:Default: ``0.0`` + +``osd_mclock_cost_per_byte_usec_hdd`` + +:Description: Cost per byte in microseconds to consider per OSD (for rotational + media) + +:Type: Float +:Default: ``5.2`` + +``osd_mclock_cost_per_byte_usec_ssd`` + +:Description: Cost per byte in microseconds to consider per OSD (for solid state + media) + +:Type: Float +:Default: ``0.011`` + diff --git a/ceph/doc/rados/configuration/osd-config-ref.rst b/ceph/doc/rados/configuration/osd-config-ref.rst index ded20df8b..7f69b5e80 100644 --- a/ceph/doc/rados/configuration/osd-config-ref.rst +++ b/ceph/doc/rados/configuration/osd-config-ref.rst @@ -569,8 +569,8 @@ Operations QoS Based on mClock ------------------- -Ceph's use of mClock is currently experimental and should -be approached with an exploratory mindset. +Ceph's use of mClock is now more refined and can be used by following the +steps as described in `mClock Config Reference`_. Core Concepts ````````````` @@ -597,7 +597,7 @@ words, the share of each type of service is controlled by three tags: #. weight: the proportional share of capacity if extra capacity or system oversubscribed. -In Ceph operations are graded with "cost". And the resources allocated +In Ceph, operations are graded with "cost". And the resources allocated for serving various services are consumed by these "costs". So, for example, the more reservation a services has, the more resource it is guaranteed to possess, as long as it requires. Assuming there are 2 @@ -619,10 +619,9 @@ competitor "1". In the case of client ops, it is not clamped by the limit setting, so it can make use of all the resources if there is no recovery ongoing. -CURRENT IMPLEMENTATION NOTE: the current experimental implementation -does not enforce the limit values. As a first approximation we decided -not to prevent operations that would otherwise enter the operation -sequencer from doing so. +CURRENT IMPLEMENTATION NOTE: the current implementation enforces the limit +values. Therefore, if a service crosses the enforced limit, the op remains +in the operation queue until the limit is restored. Subtleties of mClock ```````````````````` @@ -644,7 +643,7 @@ means if *W* is sufficiently large and therefore *1/W* is sufficiently small, the calculated tag may never be assigned as it will get a value of the current time. The ultimate lesson is that values for weight should not be too large. They should be under the number of requests -one expects to ve serviced each second. +one expects to be serviced each second. Caveats ``````` @@ -1125,3 +1124,4 @@ Miscellaneous .. _Pool & PG Config Reference: ../pool-pg-config-ref .. _Journal Config Reference: ../journal-ref .. _cache target dirty high ratio: ../../operations/pools#cache-target-dirty-high-ratio +.. _mClock Config Reference: ../mclock-config-ref diff --git a/ceph/doc/rbd/rbd-live-migration.rst b/ceph/doc/rbd/rbd-live-migration.rst index fef951fb3..c3e09193d 100644 --- a/ceph/doc/rbd/rbd-live-migration.rst +++ b/ceph/doc/rbd/rbd-live-migration.rst @@ -116,7 +116,7 @@ The import-only live-migration process is initiated by running the same and providing a JSON-encoded ``source-spec`` to describe how to access the source image data. This ``source-spec`` can either be passed directly via the `--source-spec` optional, or via a file or STDIN via the -`--source-spec-file` optional:: +`--source-spec-path` optional:: $ rbd migration prepare --import-only --source-spec "" migration_target diff --git a/ceph/doc/rbd/rbd-persistent-write-back-cache.rst b/ceph/doc/rbd/rbd-persistent-write-back-cache.rst index e8ecf4a81..db91e0697 100644 --- a/ceph/doc/rbd/rbd-persistent-write-back-cache.rst +++ b/ceph/doc/rbd/rbd-persistent-write-back-cache.rst @@ -47,8 +47,8 @@ need to be enabled.:: rbd persistent cache mode = {cache-mode} rbd plugins = pwl_cache -Value of {cache-mode} can be ``rwl`` or ``ssd``. By default it is -``disabled`` +Value of {cache-mode} can be ``rwl``, ``ssd`` or ``disabled``. By default the +cache is disabled. Here are some cache configuration settings: @@ -56,10 +56,12 @@ Here are some cache configuration settings: have DAX enabled (see `DAX`_) when using ``rwl`` mode to avoid performance degradation. -- ``rbd_persistent_cache_size`` The cache size per image. +- ``rbd_persistent_cache_size`` The cache size per image. The minimum cache + size is 1 GB. - ``rbd_persistent_cache_log_periodic_stats`` This is a debug option. It is - used to emit periodic perf stats to the debug log. + used to emit periodic perf stats to the debug log if ``debug rbd pwl`` is + set to ``1`` or higher. The above configurations can be set per-host, per-pool, per-image etc. Eg, to set per-host, add the overrides to the appropriate `section`_ in the host's @@ -70,21 +72,21 @@ Cache Status ------------ The persistent write-back cache is enabled when the exclusive lock is acquired, -and it is closed when the exclusive lock is released. To check the transient -cache status, users may use the command ``rbd status``. :: +and it is closed when the exclusive lock is released. To check the cache status, +users may use the command ``rbd status``. :: rbd status {pool-name}/{image-name} The status of the cache is shown, including present, clean, cache size and the -position. +location. Currently the status is updated only at the time the cache is opened +and closed and therefore may appear to be out of date (e.g. show that the cache +is clean when it is actually dirty). For example:: $ rbd status rbd/foo Watchers: none - image cache state: - clean: false size: 1 GiB host: sceph9 path: /tmp - + Image cache state: {"present":"true","empty":"false","clean":"true","cache_type":"ssd","pwl_host":"sceph9","pwl_path":"/tmp/rbd-pwl.rbd.abcdef123456.pool","pwl_size":1073741824} Discard Cache ------------- diff --git a/ceph/make-dist b/ceph/make-dist index 991c1bf90..2e97a1a12 100755 --- a/ceph/make-dist +++ b/ceph/make-dist @@ -1,7 +1,21 @@ #!/bin/bash -e +SCRIPTNAME="$(basename "${0}")" +BASEDIR="$(readlink -f "$(dirname "${0}")")" + if [ ! -d .git ]; then - echo "no .git present. run this from the base dir of the git checkout." + echo "$SCRIPTNAME: Full path to the script: $BASEDIR/$SCRIPTNAME" + echo "$SCRIPTNAME: No .git present. Run this from the base dir of the git checkout." + exit 1 +fi + +# Running the script from a directory containing a colon anywhere in the path +# will expose us to the dreaded "[BUG] npm run [command] failed if the directory +# path contains colon" bug https://github.com/npm/cli/issues/633 +# (see https://tracker.ceph.com/issues/39556 for details) +if [[ "$BASEDIR" == *:* ]] ; then + echo "$SCRIPTNAME: Full path to the script: $BASEDIR/$SCRIPTNAME" + echo "$SCRIPTNAME: The path to the script contains a colon. Their presence has been known to break the script." exit 1 fi diff --git a/ceph/monitoring/grafana/dashboards/host-details.json b/ceph/monitoring/grafana/dashboards/host-details.json index 0300e4f01..237349dd3 100644 --- a/ceph/monitoring/grafana/dashboards/host-details.json +++ b/ceph/monitoring/grafana/dashboards/host-details.json @@ -37,7 +37,7 @@ "gnetId": null, "graphTooltip": 0, "id": null, - "iteration": 1557386759572, + "iteration": 1615564911000, "links": [], "panels": [ { @@ -182,7 +182,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (mode) (\n irate(node_cpu{instance=~\"($ceph_hosts).*\", mode=~\"(irq|nice|softirq|steal|system|user|iowait)\"}[1m]) or\n irate(node_cpu_seconds_total{instance=~\"($ceph_hosts).*\", mode=~\"(irq|nice|softirq|steal|system|user|iowait)\"}[1m])\n) / scalar(\n sum(irate(node_cpu{instance=~\"($ceph_hosts).*\"}[1m]) or\n irate(node_cpu_seconds_total{instance=~\"($ceph_hosts).*\"}[1m]))\n) * 100", + "expr": "sum by (mode) (\n irate(node_cpu{instance=~\"($ceph_hosts)([\\\\.:].*)?\", mode=~\"(irq|nice|softirq|steal|system|user|iowait)\"}[1m]) or\n irate(node_cpu_seconds_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\", mode=~\"(irq|nice|softirq|steal|system|user|iowait)\"}[1m])\n) / scalar(\n sum(irate(node_cpu{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[1m]) or\n irate(node_cpu_seconds_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[1m]))\n) * 100", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{mode}}", @@ -283,14 +283,14 @@ "steppedLine": false, "targets": [ { - "expr": "(node_memory_MemTotal{instance=~\"[[ceph_hosts]].*\"} or node_memory_MemTotal_bytes{instance=~\"[[ceph_hosts]].*\"})- (\n (node_memory_MemFree{instance=~\"[[ceph_hosts]].*\"} or node_memory_MemFree_bytes{instance=~\"[[ceph_hosts]].*\"}) + \n (node_memory_Cached{instance=~\"[[ceph_hosts]].*\"} or node_memory_Cached_bytes{instance=~\"[[ceph_hosts]].*\"}) + \n (node_memory_Buffers{instance=~\"[[ceph_hosts]].*\"} or node_memory_Buffers_bytes{instance=~\"[[ceph_hosts]].*\"}) +\n (node_memory_Slab{instance=~\"[[ceph_hosts]].*\"} or node_memory_Slab_bytes{instance=~\"[[ceph_hosts]].*\"})\n )\n \n", + "expr": "(node_memory_MemTotal{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"} or node_memory_MemTotal_bytes{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"})- (\n (node_memory_MemFree{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"} or node_memory_MemFree_bytes{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"}) + \n (node_memory_Cached{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"} or node_memory_Cached_bytes{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"}) + \n (node_memory_Buffers{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"} or node_memory_Buffers_bytes{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"}) +\n (node_memory_Slab{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"} or node_memory_Slab_bytes{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"})\n )\n \n", "format": "time_series", "intervalFactor": 1, "legendFormat": "used", "refId": "D" }, { - "expr": "node_memory_MemFree{instance=~\"[[ceph_hosts]].*\"} or node_memory_MemFree_bytes{instance=~\"[[ceph_hosts]].*\"} ", + "expr": "node_memory_MemFree{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"} or node_memory_MemFree_bytes{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"} ", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -298,7 +298,7 @@ "refId": "A" }, { - "expr": "(node_memory_Cached{instance=~\"[[ceph_hosts]].*\"} or node_memory_Cached_bytes{instance=~\"[[ceph_hosts]].*\"}) + \n(node_memory_Buffers{instance=~\"[[ceph_hosts]].*\"} or node_memory_Buffers_bytes{instance=~\"[[ceph_hosts]].*\"}) +\n(node_memory_Slab{instance=~\"[[ceph_hosts]].*\"} or node_memory_Slab_bytes{instance=~\"[[ceph_hosts]].*\"}) \n", + "expr": "(node_memory_Cached{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"} or node_memory_Cached_bytes{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"}) + \n(node_memory_Buffers{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"} or node_memory_Buffers_bytes{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"}) +\n(node_memory_Slab{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"} or node_memory_Slab_bytes{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"}) \n", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -306,7 +306,7 @@ "refId": "C" }, { - "expr": "node_memory_MemTotal{instance=~\"[[ceph_hosts]].*\"} or node_memory_MemTotal_bytes{instance=~\"[[ceph_hosts]].*\"} ", + "expr": "node_memory_MemTotal{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"} or node_memory_MemTotal_bytes{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"} ", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -401,7 +401,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (device) (\n irate(node_network_receive_bytes{instance=~\"($ceph_hosts).*\",device!=\"lo\"}[1m]) or \n irate(node_network_receive_bytes_total{instance=~\"($ceph_hosts).*\",device!=\"lo\"}[1m])\n)", + "expr": "sum by (device) (\n irate(node_network_receive_bytes{instance=~\"($ceph_hosts)([\\\\.:].*)?\",device!=\"lo\"}[1m]) or \n irate(node_network_receive_bytes_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\",device!=\"lo\"}[1m])\n)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}}.rx", @@ -410,7 +410,7 @@ "textEditor": true }, { - "expr": "sum by (device) (\n irate(node_network_transmit_bytes{instance=~\"($ceph_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_transmit_bytes_total{instance=~\"($ceph_hosts).*\",device!=\"lo\"}[1m])\n)", + "expr": "sum by (device) (\n irate(node_network_transmit_bytes{instance=~\"($ceph_hosts)([\\\\.:].*)?\",device!=\"lo\"}[1m]) or\n irate(node_network_transmit_bytes_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\",device!=\"lo\"}[1m])\n)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}}.tx", @@ -501,7 +501,7 @@ "steppedLine": false, "targets": [ { - "expr": "irate(node_network_receive_drop{instance=~\"[[ceph_hosts]].*\"}[1m]) or irate(node_network_receive_drop_total{instance=~\"[[ceph_hosts]].*\"}[1m])", + "expr": "irate(node_network_receive_drop{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"}[1m]) or irate(node_network_receive_drop_total{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"}[1m])", "format": "time_series", "instant": false, "intervalFactor": 1, @@ -509,7 +509,7 @@ "refId": "A" }, { - "expr": "irate(node_network_transmit_drop{instance=~\"[[ceph_hosts]].*\"}[1m]) or irate(node_network_transmit_drop_total{instance=~\"[[ceph_hosts]].*\"}[1m])", + "expr": "irate(node_network_transmit_drop{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"}[1m]) or irate(node_network_transmit_drop_total{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"}[1m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}}.tx", @@ -621,7 +621,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(ceph_osd_stat_bytes and on (ceph_daemon) ceph_disk_occupation{instance=~\"($ceph_hosts).*\"})", + "expr": "sum(ceph_osd_stat_bytes and on (ceph_daemon) ceph_disk_occupation{instance=~\"($ceph_hosts)([\\\\.:].*)?\"})", "format": "time_series", "intervalFactor": 2, "refId": "A", @@ -685,7 +685,7 @@ "steppedLine": false, "targets": [ { - "expr": "irate(node_network_receive_errs{instance=~\"[[ceph_hosts]].*\"}[1m]) or irate(node_network_receive_errs_total{instance=~\"[[ceph_hosts]].*\"}[1m])", + "expr": "irate(node_network_receive_errs{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"}[1m]) or irate(node_network_receive_errs_total{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"}[1m])", "format": "time_series", "instant": false, "intervalFactor": 1, @@ -693,7 +693,7 @@ "refId": "A" }, { - "expr": "irate(node_network_transmit_errs{instance=~\"[[ceph_hosts]].*\"}[1m]) or irate(node_network_transmit_errs_total{instance=~\"[[ceph_hosts]].*\"}[1m])", + "expr": "irate(node_network_transmit_errs{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"}[1m]) or irate(node_network_transmit_errs_total{instance=~\"[[ceph_hosts]]([\\\\.:].*)?\"}[1m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}}.tx", @@ -798,7 +798,7 @@ "steppedLine": false, "targets": [ { - "expr": "label_replace(\n (\n irate(node_disk_writes_completed{instance=~\"($ceph_hosts).*\"}[5m]) or\n irate(node_disk_writes_completed_total{instance=~\"($ceph_hosts).*\"}[5m])\n ),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n)\n* on(instance, device) group_left(ceph_daemon)\n label_replace(\n label_replace(\n ceph_disk_occupation,\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n )", + "expr": "label_replace(\n (\n irate(node_disk_writes_completed{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) or\n irate(node_disk_writes_completed_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m])\n ),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n)\n* on(instance, device, ceph_daemon) group_left\n label_replace(\n label_replace(\n ceph_disk_occupation,\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n )", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}}({{ceph_daemon}}) writes", @@ -807,7 +807,7 @@ "textEditor": true }, { - "expr": "label_replace(\n (irate(node_disk_reads_completed{instance=~\"($ceph_hosts).*\"}[5m]) or irate(node_disk_reads_completed_total{instance=~\"($ceph_hosts).*\"}[5m])),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n)\n* on(instance, device) group_left(ceph_daemon)\n label_replace(\n label_replace(\n ceph_disk_occupation,\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n )", + "expr": "label_replace(\n (irate(node_disk_reads_completed{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) or irate(node_disk_reads_completed_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m])),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n)\n* on(instance, device, ceph_daemon) group_left\n label_replace(\n label_replace(\n ceph_disk_occupation,\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n )", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -899,14 +899,14 @@ "steppedLine": false, "targets": [ { - "expr": "label_replace((irate(node_disk_bytes_written{instance=~\"($ceph_hosts).*\"}[5m]) or irate(node_disk_written_bytes_total{instance=~\"($ceph_hosts).*\"}[5m])), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") * on(instance, device) group_left(ceph_daemon) label_replace(label_replace(ceph_disk_occupation, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", + "expr": "label_replace((irate(node_disk_bytes_written{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) or irate(node_disk_written_bytes_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m])), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") * on(instance, device, ceph_daemon) group_left label_replace(label_replace(ceph_disk_occupation, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}}({{ceph_daemon}}) write", "refId": "B" }, { - "expr": "label_replace((irate(node_disk_bytes_read{instance=~\"($ceph_hosts).*\"}[5m]) or irate(node_disk_read_bytes_total{instance=~\"($ceph_hosts).*\"}[5m])), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") * on(instance, device) group_left(ceph_daemon) label_replace(label_replace(ceph_disk_occupation, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", + "expr": "label_replace((irate(node_disk_bytes_read{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) or irate(node_disk_read_bytes_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m])), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") * on(instance, device, ceph_daemon) group_left label_replace(label_replace(ceph_disk_occupation, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}}({{ceph_daemon}}) read", @@ -992,7 +992,7 @@ "steppedLine": false, "targets": [ { - "expr": "max by(instance,device) (label_replace((irate(node_disk_write_time_seconds_total{ instance=~\"($ceph_hosts).*\"}[5m]) ) / clamp_min(irate(node_disk_writes_completed_total{ instance=~\"($ceph_hosts).*\"}[5m]), 0.001) or (irate(node_disk_read_time_seconds_total{ instance=~\"($ceph_hosts).*\"}[5m]) ) / clamp_min(irate(node_disk_reads_completed_total{ instance=~\"($ceph_hosts).*\"}[5m]), 0.001), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")) * on(instance,device) group_left(ceph_daemon) label_replace(label_replace(ceph_disk_occupation{instance=~\"($ceph_hosts).*\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", + "expr": "max by(instance,device) (label_replace((irate(node_disk_write_time_seconds_total{ instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) ) / clamp_min(irate(node_disk_writes_completed_total{ instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]), 0.001) or (irate(node_disk_read_time_seconds_total{ instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) ) / clamp_min(irate(node_disk_reads_completed_total{ instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]), 0.001), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")) * on(instance, device, ceph_daemon) group_left label_replace(label_replace(ceph_disk_occupation{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1083,7 +1083,7 @@ "steppedLine": false, "targets": [ { - "expr": "label_replace(((irate(node_disk_io_time_ms{instance=~\"($ceph_hosts).*\"}[5m]) / 10 ) or irate(node_disk_io_time_seconds_total{instance=~\"($ceph_hosts).*\"}[5m]) * 100), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") * on(instance, device) group_left(ceph_daemon) label_replace(label_replace(ceph_disk_occupation{instance=~\"($ceph_hosts).*\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", + "expr": "label_replace(((irate(node_disk_io_time_ms{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) / 10 ) or irate(node_disk_io_time_seconds_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) * 100), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") * on(instance, device, ceph_daemon) group_left label_replace(label_replace(ceph_disk_occupation{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", "format": "time_series", "hide": false, "intervalFactor": 1, diff --git a/ceph/monitoring/grafana/dashboards/hosts-overview.json b/ceph/monitoring/grafana/dashboards/hosts-overview.json index 804aa51cc..b179d5717 100644 --- a/ceph/monitoring/grafana/dashboards/hosts-overview.json +++ b/ceph/monitoring/grafana/dashboards/hosts-overview.json @@ -131,7 +131,6 @@ "#d44a3a" ], "datasource": "$datasource", - "decimals": 0, "description": "Average CPU busy across all hosts (OSD, RGW, MON etc) within the cluster", "decimals": 2, "format": "percentunit", @@ -215,7 +214,6 @@ "#d44a3a" ], "datasource": "$datasource", - "decimals": 0, "description": "Average Memory Usage across all hosts in the cluster (excludes buffer/cache usage)", "decimals": 2, "format": "percentunit", @@ -433,7 +431,7 @@ "tableColumn": "", "targets": [ { - "expr" : "avg (\n label_replace((irate(node_disk_io_time_ms[5m]) / 10 ) or\n (irate(node_disk_io_time_seconds_total[5m]) * 100), \"instance\", \"$1\", \"instance\", \"([^.:]*).*\"\n ) *\n on(instance, device) label_replace(label_replace(ceph_disk_occupation{instance=~\"($osd_hosts).*\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^.:]*).*\")\n)", + "expr" : "avg (\n label_replace((irate(node_disk_io_time_ms[5m]) / 10 ) or\n (irate(node_disk_io_time_seconds_total[5m]) * 100), \"instance\", \"$1\", \"instance\", \"([^.:]*).*\"\n ) *\n on(instance, device, ceph_daemon) label_replace(label_replace(ceph_disk_occupation{instance=~\"($osd_hosts).*\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^.:]*).*\")\n)", "format": "time_series", "instant": true, "intervalFactor": 1, diff --git a/ceph/monitoring/grafana/dashboards/pool-overview.json b/ceph/monitoring/grafana/dashboards/pool-overview.json index d8654599a..cd699348b 100644 --- a/ceph/monitoring/grafana/dashboards/pool-overview.json +++ b/ceph/monitoring/grafana/dashboards/pool-overview.json @@ -15,223 +15,713 @@ "editable": false, "gnetId": null, "graphTooltip": 0, - "iteration": 1551789900270, + "id": null, + "iteration": 1617656284287, "links": [], "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], "datasource": "$datasource", - "fill": 1, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, "gridPos": { - "h": 7, - "w": 12, + "h": 3, + "w": 3, "x": 0, - "y": 1 + "y": 0 }, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null }, - "lines": true, - "linewidth": 1, + "tableColumn": "", + "targets": [ + { + "expr": "count(ceph_pool_metadata)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Pools", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "description": "Count of the pools that have compression enabled", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 0 + }, + "id": 7, + "interval": null, "links": [], - "minSpan": 12, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, { - "alias": "/.* read/", - "transform": "negative-Y" + "name": "range to text", + "value": 2 } ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.7.4", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", "targets": [ { - "expr": "topk($topk,rate(ceph_pool_rd[1m]) + on(pool_id) group_left(instance,name) ceph_pool_metadata) ", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{name}} - read", - "refId": "F" + "expr": "count(ceph_pool_metadata{compression_mode!=\"none\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Pools with Compression", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 1, + "description": "Total raw capacity available to the cluster", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 0 + }, + "id": 27, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, { - "expr": "topk($topk,rate(ceph_pool_wr[1m]) + on(pool_id) group_left(instance,name) ceph_pool_metadata) ", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{name}} - write", + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(ceph_osd_stat_bytes)", + "interval": "", + "legendFormat": "", "refId": "A" } ], - "thresholds": [], + "thresholds": "", "timeFrom": null, "timeShift": null, - "title": "Top $topk Client IOPS by Pool", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" + "title": "Total Raw Capacity", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "description": "Total raw capacity consumed by user data and associated overheads (metadata + redundancy)", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "gridPos": { + "h": 3, + "w": 3, + "x": 9, + "y": 0 }, - "yaxes": [ + "id": 25, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ { - "format": "none", - "label": "Read (-) / Write (+)", - "logBase": 1, - "max": null, - "min": null, - "show": true + "name": "value to text", + "value": 1 }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "name": "range to text", + "value": 2 } ], - "yaxis": { - "align": false, - "alignLevel": null - } + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(ceph_pool_bytes_used)", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Raw Capacity Consumed", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current", + "decimals": 2 }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], "datasource": "$datasource", - "fill": 1, + "decimals": 1, + "description": "Total of client data stored in the cluster", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, "gridPos": { - "h": 7, - "w": 12, + "h": 3, + "w": 3, "x": 12, - "y": 1 + "y": 0 }, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, + "id": 23, + "interval": null, "links": [], - "minSpan": 12, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + "mappingType": 1, + "mappingTypes": [ { - "alias": "/.* read/", - "transform": "negative-Y" + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 } ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", "targets": [ { - "expr": "topk($topk,rate(ceph_pool_rd_bytes[1m]) + on(pool_id) group_left(instance,name) ceph_pool_metadata)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{name}} - read", - "refId": "A", - "textEditor": true + "expr": "sum(ceph_pool_stored)", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Logical Stored ", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 1, + "description": "A compression saving is determined as the data eligible to be compressed minus the capacity used to store the data after compression", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 15, + "y": 0 + }, + "id": 9, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, { - "expr": "topk($topk,rate(ceph_pool_wr_bytes[1m]) + on(pool_id) group_left(instance,name) ceph_pool_metadata)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{name}} - write", - "refId": "B" + "name": "range to text", + "value": 2 } ], - "thresholds": [], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(ceph_pool_compress_under_bytes - ceph_pool_compress_bytes_used)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", "timeFrom": null, "timeShift": null, - "title": "Top $topk Client Throughput by Pool", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" + "title": "Compression Savings", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "description": "Indicates how suitable the data is within the pools that are/have been enabled for compression - averaged across all pools holding compressed data\n", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "gridPos": { + "h": 3, + "w": 3, + "x": 18, + "y": 0 }, - "yaxes": [ + "id": 17, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ { - "format": "Bps", - "label": "Read (-) / Writes (+)", - "logBase": 1, - "max": null, - "min": null, - "show": true + "name": "value to text", + "value": 1 }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "name": "range to text", + "value": 2 } ], - "yaxis": { - "align": false, - "alignLevel": null - } + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(ceph_pool_compress_under_bytes > 0) / sum(ceph_pool_stored_raw and ceph_pool_compress_under_bytes > 0)) * 100", + "format": "table", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Compression Eligibility", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" }, { - "columns": [ + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "description": "This factor describes the average ratio of data eligible to be compressed divided by the data actually stored. It does not account for data written that was ineligible for compression (too small, or compression yield too low)", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 15, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "80%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(ceph_pool_compress_under_bytes > 0) / sum(ceph_pool_compress_bytes_used > 0)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Compression Factor", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ { - "text": "Current", - "value": "current" + "op": "=", + "text": "N/A", + "value": "null" } ], + "valueName": "current" + }, + { + "columns": [], "datasource": "$datasource", "fontSize": "100%", "gridPos": { - "h": 7, - "w": 8, + "h": 6, + "w": 24, "x": 0, - "y": 9 + "y": 3 }, - "id": 3, + "id": 5, "links": [], - "minSpan": 12, + "maxPerRow": 3, "pageSize": null, "scroll": true, "showHeader": true, @@ -241,7 +731,72 @@ }, "styles": [ { - "alias": "", + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "instance", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "job", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Pool Name", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Pool ID", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -249,14 +804,15 @@ "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "Time", + "decimals": 0, + "pattern": "pool_id", "thresholds": [], "type": "hidden", - "unit": "short" + "unit": "none" }, { - "alias": "", + "alias": "Compression Factor", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -264,14 +820,36 @@ "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "id", + "decimals": 1, + "mappingType": 1, + "pattern": "Value #A", "thresholds": [], - "type": "hidden", - "unit": "short" + "type": "number", + "unit": "none" }, { - "alias": "", + "alias": "% Used", + "align": "auto", + "colorMode": "value", + "colors": [ + "rgb(0, 0, 0)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #D", + "thresholds": [ + "70", + "85" + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Usable Free", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -280,13 +858,15 @@ ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, - "pattern": "instance", + "mappingType": 1, + "pattern": "Value #B", "thresholds": [], - "type": "hidden", - "unit": "short" + "type": "number", + "unit": "bytes" }, { - "alias": "", + "alias": "Compression Eligibility", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -294,14 +874,16 @@ "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "job", + "decimals": 0, + "mappingType": 1, + "pattern": "Value #C", "thresholds": [], - "type": "hidden", - "unit": "short" + "type": "number", + "unit": "percent" }, { - "alias": "Pool Name", + "alias": "Compression Savings", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -309,15 +891,17 @@ "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "name", + "decimals": 1, + "mappingType": 1, + "pattern": "Value #E", "thresholds": [], "type": "number", - "unit": "short" + "unit": "bytes" }, { - "alias": "Pool ID", - "colorMode": null, + "alias": "Growth (5d)", + "align": "auto", + "colorMode": "value", "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", @@ -325,13 +909,18 @@ ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, - "pattern": "pool_id", - "thresholds": [], + "mappingType": 1, + "pattern": "Value #F", + "thresholds": [ + "0", + "0" + ], "type": "number", - "unit": "short" + "unit": "bytes" }, { - "alias": "IOPS (R+W)", + "alias": "IOPS", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -340,60 +929,32 @@ ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 0, - "pattern": "Value", + "mappingType": 1, + "pattern": "Value #G", "thresholds": [], "type": "number", "unit": "none" - } - ], - "targets": [ - { - "expr": "topk($topk,((irate(ceph_pool_rd[1m]) + irate(ceph_pool_wr[1m])) + on(pool_id) group_left(instance,name) ceph_pool_metadata))", - "format": "table", - "instant": true, - "intervalFactor": 2, - "refId": "A", - "textEditor": true - } - ], - "title": "Top $topk Pools by Client IOPS", - "transform": "table", - "type": "table" - }, - { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": "$datasource", - "fontSize": "100%", - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 9 - }, - "id": 4, - "links": [], - "minSpan": 12, - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 5, - "desc": true - }, - "styles": [ + }, { - "alias": "Time", + "alias": "Bandwidth", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" + "decimals": 0, + "mappingType": 1, + "pattern": "Value #H", + "thresholds": [], + "type": "number", + "unit": "Bps" }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -402,13 +963,15 @@ ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, - "pattern": "id", + "mappingType": 1, + "pattern": "__name__", "thresholds": [], "type": "hidden", "unit": "short" }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -417,13 +980,15 @@ ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, - "pattern": "instance", + "mappingType": 1, + "pattern": "type", "thresholds": [], "type": "hidden", "unit": "short" }, { "alias": "", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -432,13 +997,15 @@ ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, - "pattern": "job", + "mappingType": 1, + "pattern": "compression_mode", "thresholds": [], "type": "hidden", "unit": "short" }, { - "alias": "Pool Name", + "alias": "Type", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -447,13 +1014,15 @@ ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, - "pattern": "name", + "mappingType": 1, + "pattern": "description", "thresholds": [], - "type": "number", + "type": "string", "unit": "short" }, { - "alias": "Pool ID", + "alias": "Stored", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -461,14 +1030,33 @@ "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "pool_id", + "decimals": 1, + "mappingType": 1, + "pattern": "Value #J", "thresholds": [], "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #I", + "thresholds": [], + "type": "hidden", "unit": "short" }, { - "alias": "Throughput", + "alias": "Compression", + "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", @@ -477,170 +1065,437 @@ ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, - "pattern": "Value", + "mappingType": 1, + "pattern": "Value #K", "thresholds": [], - "type": "number", - "unit": "decbytes" + "type": "string", + "unit": "short", + "valueMaps": [ + { + "text": "ON", + "value": "1" + } + ] + } + ], + "targets": [ + { + "expr": "(ceph_pool_percent_used * on(pool_id) group_left(name) ceph_pool_metadata)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "D" + }, + { + "expr": "ceph_pool_stored * on(pool_id) group_left ceph_pool_metadata", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "J" + }, + { + "expr": "ceph_pool_max_avail * on(pool_id) group_left(name) ceph_pool_metadata", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "expr": "delta(ceph_pool_stored[5d])", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "F" + }, + { + "expr": "ceph_pool_metadata", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "I" + }, + { + "expr": "ceph_pool_metadata{compression_mode!=\"none\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "K" + }, + { + "expr": "(ceph_pool_compress_under_bytes / ceph_pool_compress_bytes_used > 0) and on(pool_id) (((ceph_pool_compress_under_bytes > 0) / ceph_pool_stored_raw) * 100 > 0.5)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + }, + { + "expr": "((ceph_pool_compress_under_bytes > 0) / ceph_pool_stored_raw) * 100", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "expr": "(ceph_pool_compress_under_bytes - ceph_pool_compress_bytes_used > 0)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "E" + }, + { + "expr": "rate(ceph_pool_rd[30s]) + rate(ceph_pool_wr[30s])", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "G" + }, + { + "expr": "rate(ceph_pool_rd_bytes[30s]) + rate(ceph_pool_wr_bytes[30s])", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "H" + }, + { + "expr": "", + "interval": "", + "legendFormat": "", + "refId": "L" } ], + "title": "Pool Overview", + "transform": "table", + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "This chart shows the sum of read and write IOPS from all clients by pool", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "topk($topk,(irate(ceph_pool_rd_bytes[1m]) + irate(ceph_pool_wr_bytes[1m])) + on(pool_id) group_left(instance,name) ceph_pool_metadata) ", - "format": "table", - "instant": true, - "intervalFactor": 2, - "refId": "A", - "textEditor": true + "expr": "topk($topk,round((rate(ceph_pool_rd[30s]) + rate(ceph_pool_wr[30s])),1) * on(pool_id) group_left(instance,name) ceph_pool_metadata) ", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{name}} ", + "refId": "F" + }, + { + "expr": "topk($topk,rate(ceph_pool_wr[30s]) + on(pool_id) group_left(instance,name) ceph_pool_metadata) ", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{name}} - write", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Top $topk Client IOPS by Pool", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "IOPS", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false } ], - "title": "Top $topk Pools by Throughput", - "transform": "table", - "type": "table" + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "columns": [], + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "$datasource", - "fontSize": "100%", + "description": "The chart shows the sum of read and write bytes from all clients, by pool", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 7, - "w": 8, - "x": 16, + "h": 8, + "w": 12, + "x": 12, "y": 9 }, - "id": 5, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, "links": [], - "minSpan": 8, - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 5, - "desc": true + "maxPerRow": 2, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] }, - "styles": [ - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "Time", - "thresholds": [], - "type": "hidden", - "unit": "short" - }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "instance", - "thresholds": [], - "type": "hidden", - "unit": "short" - }, + "expr": "topk($topk,(rate(ceph_pool_rd_bytes[30s]) + rate(ceph_pool_wr_bytes[30s])) * on(pool_id) group_left(instance,name) ceph_pool_metadata)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A", + "textEditor": true + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Top $topk Client Bandwidth by Pool", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "job", - "thresholds": [], - "type": "hidden", - "unit": "short" + "format": "Bps", + "label": "Throughput", + "logBase": 1, + "max": null, + "min": "0", + "show": true }, { - "alias": "Pool Name", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "name", - "thresholds": [], - "type": "string", - "unit": "short" - }, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Historical view of capacity usage, to help identify growth and trends in pool consumption", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ { - "alias": "Pool ID", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "pool_id", - "thresholds": [], - "type": "number", - "unit": "short" - }, + "expr": "ceph_pool_bytes_used * on(pool_id) group_right ceph_pool_metadata", + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": "14d", + "timeRegions": [ { - "alias": "Capacity Used", - "colorMode": "value", - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "Value", - "thresholds": [ - "70", - "85" - ], - "type": "number", - "unit": "percentunit" + "colorMode": "background6", + "fill": true, + "fillColor": "rgba(234, 112, 112, 0.12)", + "line": false, + "lineColor": "rgba(237, 46, 24, 0.60)", + "op": "time" } ], - "targets": [ + "timeShift": null, + "title": "Pool Capacity Usage (RAW)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ { - "expr": "topk($topk,((ceph_pool_stored / (ceph_pool_stored + ceph_pool_max_avail)) * on(pool_id) group_left(name) ceph_pool_metadata))", - "format": "table", - "hide": false, - "instant": true, - "intervalFactor": 1, - "legendFormat": "", - "refId": "D" + "decimals": 1, + "format": "bytes", + "label": "Capacity Used", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } ], - "title": "Top $topk Pools By Capacity Used", - "transform": "table", - "type": "table" + "yaxis": { + "align": false, + "alignLevel": null + } } ], "refresh": "15s", - "schemaVersion": 16, + "schemaVersion": 22, "style": "dark", "tags": [], "templating": { "list": [ { "current": { - "text": "Prometheus admin.virt1.home.fajerski.name:9090", - "value": "Prometheus admin.virt1.home.fajerski.name:9090" + "selected": false, + "text": "Dashboard1", + "value": "Dashboard1" }, "hide": 0, + "includeAll": false, "label": "Data Source", + "multi": false, "name": "datasource", "options": [], "query": "prometheus", @@ -702,5 +1557,8 @@ "timezone": "browser", "title": "Ceph Pools Overview", "uid": "z99hzWtmk", - "version": 1 + "variables": { + "list": [] + }, + "version": 10 } diff --git a/ceph/qa/cephfs/clusters/1-node-1-mds-1-osd.yaml b/ceph/qa/cephfs/clusters/1-node-1-mds-1-osd.yaml new file mode 100644 index 000000000..865b976c6 --- /dev/null +++ b/ceph/qa/cephfs/clusters/1-node-1-mds-1-osd.yaml @@ -0,0 +1,8 @@ +roles: +- [mon.a, mgr.x, mds.a, osd.0, client.0] +openstack: +- volumes: # attached to each instance + count: 1 + size: 5 # GB +- machine: + disk: 10 # GB diff --git a/ceph/qa/distros/all/rhel_8.4.yaml b/ceph/qa/distros/all/rhel_8.4.yaml new file mode 100644 index 000000000..b0b74874f --- /dev/null +++ b/ceph/qa/distros/all/rhel_8.4.yaml @@ -0,0 +1,6 @@ +os_type: rhel +os_version: "8.4" +overrides: + selinux: + whitelist: + - scontext=system_u:system_r:logrotate_t:s0 diff --git a/ceph/qa/distros/all/rhel_8.yaml b/ceph/qa/distros/all/rhel_8.yaml index 9e5fa1165..c7867a423 120000 --- a/ceph/qa/distros/all/rhel_8.yaml +++ b/ceph/qa/distros/all/rhel_8.yaml @@ -1 +1 @@ -rhel_8.3.yaml \ No newline at end of file +rhel_8.4.yaml \ No newline at end of file diff --git a/ceph/qa/distros/podman/centos_8.2_kubic_stable.yaml b/ceph/qa/distros/podman/centos_8.2_kubic_stable.yaml index f3da1c754..22fbc1997 100644 --- a/ceph/qa/distros/podman/centos_8.2_kubic_stable.yaml +++ b/ceph/qa/distros/podman/centos_8.2_kubic_stable.yaml @@ -8,9 +8,11 @@ overrides: tasks: - pexec: all: + - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module disable container-tools - sudo dnf -y install 'dnf-command(copr)' - sudo dnf -y copr enable rhcontainerbot/container-selinux - sudo curl -L -o /etc/yum.repos.d/devel:kubic:libcontainers:stable.repo https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_8/devel:kubic:libcontainers:stable.repo - sudo dnf remove -y podman - sudo dnf -y install podman + - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf diff --git a/ceph/qa/distros/podman/rhel_8.3_kubic_stable.yaml b/ceph/qa/distros/podman/rhel_8.3_kubic_stable.yaml index bf921d2b4..4e50abc45 100644 --- a/ceph/qa/distros/podman/rhel_8.3_kubic_stable.yaml +++ b/ceph/qa/distros/podman/rhel_8.3_kubic_stable.yaml @@ -8,9 +8,11 @@ overrides: tasks: - pexec: all: + - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module disable container-tools - sudo dnf -y install 'dnf-command(copr)' - sudo dnf -y copr enable rhcontainerbot/container-selinux - sudo curl -L -o /etc/yum.repos.d/devel:kubic:libcontainers:stable.repo https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_8/devel:kubic:libcontainers:stable.repo - sudo dnf remove -y podman - sudo dnf -y install podman + - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf diff --git a/ceph/qa/distros/podman/ubuntu_18.04_kubic_stable.yaml b/ceph/qa/distros/podman/ubuntu_18.04_kubic_stable.yaml index f7ee5e6fd..1055bcc9e 100644 --- a/ceph/qa/distros/podman/ubuntu_18.04_kubic_stable.yaml +++ b/ceph/qa/distros/podman/ubuntu_18.04_kubic_stable.yaml @@ -5,8 +5,9 @@ os_version: "18.04" tasks: - pexec: all: + - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_18.04/Release.key | sudo apt-key add - - echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_18.04/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list - sudo apt update - sudo apt -y install podman - - echo -e "[[registry]]\nlocation = 'docker.io'\n\n[[registry.mirror]]\nlocation='docker-mirror.front.sepia.ceph.com:5000'\n" | sudo tee /etc/containers/registries.conf + - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf diff --git a/ceph/qa/distros/podman/ubuntu_20.04_kubic_stable.yaml b/ceph/qa/distros/podman/ubuntu_20.04_kubic_stable.yaml index 2eae5e2a3..3a04f50a8 100644 --- a/ceph/qa/distros/podman/ubuntu_20.04_kubic_stable.yaml +++ b/ceph/qa/distros/podman/ubuntu_20.04_kubic_stable.yaml @@ -5,8 +5,9 @@ os_version: "20.04" tasks: - pexec: all: + - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/Release.key | sudo apt-key add - - echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list - sudo apt update - sudo apt -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install podman containernetworking-plugins - - echo -e "[[registry]]\nlocation = 'docker.io'\n\n[[registry.mirror]]\nlocation='docker-mirror.front.sepia.ceph.com:5000'\n" | sudo tee /etc/containers/registries.conf + - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf diff --git a/ceph/qa/distros/podman/ubuntu_20.04_kubic_testing.yaml b/ceph/qa/distros/podman/ubuntu_20.04_kubic_testing.yaml index 1844f4ebf..56b4d1e0f 100644 --- a/ceph/qa/distros/podman/ubuntu_20.04_kubic_testing.yaml +++ b/ceph/qa/distros/podman/ubuntu_20.04_kubic_testing.yaml @@ -5,8 +5,9 @@ os_version: "20.04" tasks: - pexec: all: + - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/testing/xUbuntu_20.04/Release.key | sudo apt-key add - - echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/testing/xUbuntu_20.04/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:testing.list - sudo apt update - sudo apt -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install podman containernetworking-plugins - - echo -e "[[registry]]\nlocation = 'docker.io'\n\n[[registry.mirror]]\nlocation='docker-mirror.front.sepia.ceph.com:5000'\n" | sudo tee /etc/containers/registries.conf + - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf diff --git a/ceph/qa/distros/supported-all-distro/ubuntu_latest.yaml b/ceph/qa/distros/supported-all-distro/ubuntu_latest.yaml index 64a66d9aa..75d907e3b 120000 --- a/ceph/qa/distros/supported-all-distro/ubuntu_latest.yaml +++ b/ceph/qa/distros/supported-all-distro/ubuntu_latest.yaml @@ -1 +1 @@ -../all/ubuntu_18.04.yaml \ No newline at end of file +../all/ubuntu_20.04.yaml \ No newline at end of file diff --git a/ceph/qa/distros/supported-random-distro$/ubuntu_latest.yaml b/ceph/qa/distros/supported-random-distro$/ubuntu_latest.yaml index 64a66d9aa..75d907e3b 120000 --- a/ceph/qa/distros/supported-random-distro$/ubuntu_latest.yaml +++ b/ceph/qa/distros/supported-random-distro$/ubuntu_latest.yaml @@ -1 +1 @@ -../all/ubuntu_18.04.yaml \ No newline at end of file +../all/ubuntu_20.04.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/bugs/client_trim_caps/centos_latest.yaml b/ceph/qa/suites/fs/bugs/client_trim_caps/centos_latest.yaml new file mode 120000 index 000000000..bd9854e70 --- /dev/null +++ b/ceph/qa/suites/fs/bugs/client_trim_caps/centos_latest.yaml @@ -0,0 +1 @@ +.qa/distros/supported/centos_latest.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/dashboard/% b/ceph/qa/suites/fs/full/% similarity index 100% rename from ceph/qa/suites/rados/cephadm/dashboard/% rename to ceph/qa/suites/fs/full/% diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/.qa b/ceph/qa/suites/fs/full/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/.qa rename to ceph/qa/suites/fs/full/.qa diff --git a/ceph/qa/suites/fs/full/begin.yaml b/ceph/qa/suites/fs/full/begin.yaml new file mode 120000 index 000000000..311d404f7 --- /dev/null +++ b/ceph/qa/suites/fs/full/begin.yaml @@ -0,0 +1 @@ +.qa/cephfs/begin.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/0-size-min-size-overrides/.qa b/ceph/qa/suites/fs/full/clusters/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/0-size-min-size-overrides/.qa rename to ceph/qa/suites/fs/full/clusters/.qa diff --git a/ceph/qa/suites/fs/full/clusters/1-node-1-mds-1-osd.yaml b/ceph/qa/suites/fs/full/clusters/1-node-1-mds-1-osd.yaml new file mode 120000 index 000000000..517b76547 --- /dev/null +++ b/ceph/qa/suites/fs/full/clusters/1-node-1-mds-1-osd.yaml @@ -0,0 +1 @@ +.qa/cephfs/clusters/1-node-1-mds-1-osd.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/full/conf b/ceph/qa/suites/fs/full/conf new file mode 120000 index 000000000..16e8cc44b --- /dev/null +++ b/ceph/qa/suites/fs/full/conf @@ -0,0 +1 @@ +.qa/cephfs/conf \ No newline at end of file diff --git a/ceph/qa/suites/fs/full/distro b/ceph/qa/suites/fs/full/distro new file mode 120000 index 000000000..0862b4457 --- /dev/null +++ b/ceph/qa/suites/fs/full/distro @@ -0,0 +1 @@ +.qa/distros/supported-random-distro$ \ No newline at end of file diff --git a/ceph/qa/suites/fs/full/mount/fuse.yaml b/ceph/qa/suites/fs/full/mount/fuse.yaml new file mode 100644 index 000000000..8338cc493 --- /dev/null +++ b/ceph/qa/suites/fs/full/mount/fuse.yaml @@ -0,0 +1,2 @@ +tasks: + - ceph-fuse: diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/1-install/.qa b/ceph/qa/suites/fs/full/objectstore/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/1-install/.qa rename to ceph/qa/suites/fs/full/objectstore/.qa diff --git a/ceph/qa/suites/fs/full/objectstore/bluestore-bitmap.yaml b/ceph/qa/suites/fs/full/objectstore/bluestore-bitmap.yaml new file mode 120000 index 000000000..a59cf5175 --- /dev/null +++ b/ceph/qa/suites/fs/full/objectstore/bluestore-bitmap.yaml @@ -0,0 +1 @@ +.qa/objectstore/bluestore-bitmap.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/full/overrides.yaml b/ceph/qa/suites/fs/full/overrides.yaml new file mode 100644 index 000000000..921528d66 --- /dev/null +++ b/ceph/qa/suites/fs/full/overrides.yaml @@ -0,0 +1,19 @@ +overrides: + ceph: + conf: + mgr: + debug client: 20 + log-ignorelist: + - OSD full dropping all updates + - OSD near full + - pausewr flag + - failsafe engaged, dropping updates + - failsafe disengaged, no longer dropping + - is full \(reached quota + - POOL_FULL + - POOL_NEARFULL + - POOL_BACKFILLFULL + - PG_DEGRADED + - OSD_OUT_OF_ORDER_FULL + - OSD_NEARFULL + - OSD_FULL diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/clusters/+ b/ceph/qa/suites/fs/full/overrides/+ similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/clusters/+ rename to ceph/qa/suites/fs/full/overrides/+ diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/backoff/.qa b/ceph/qa/suites/fs/full/overrides/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/backoff/.qa rename to ceph/qa/suites/fs/full/overrides/.qa diff --git a/ceph/qa/suites/fs/full/overrides/frag_enable.yaml b/ceph/qa/suites/fs/full/overrides/frag_enable.yaml new file mode 120000 index 000000000..34a39a368 --- /dev/null +++ b/ceph/qa/suites/fs/full/overrides/frag_enable.yaml @@ -0,0 +1 @@ +.qa/cephfs/overrides/frag_enable.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/full/overrides/no_client_pidfile.yaml b/ceph/qa/suites/fs/full/overrides/no_client_pidfile.yaml new file mode 120000 index 000000000..8888f3327 --- /dev/null +++ b/ceph/qa/suites/fs/full/overrides/no_client_pidfile.yaml @@ -0,0 +1 @@ +.qa/overrides/no_client_pidfile.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/full/overrides/whitelist_health.yaml b/ceph/qa/suites/fs/full/overrides/whitelist_health.yaml new file mode 120000 index 000000000..74f39a49b --- /dev/null +++ b/ceph/qa/suites/fs/full/overrides/whitelist_health.yaml @@ -0,0 +1 @@ +.qa/cephfs/overrides/whitelist_health.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/full/overrides/whitelist_wrongly_marked_down.yaml b/ceph/qa/suites/fs/full/overrides/whitelist_wrongly_marked_down.yaml new file mode 120000 index 000000000..b4528c0f8 --- /dev/null +++ b/ceph/qa/suites/fs/full/overrides/whitelist_wrongly_marked_down.yaml @@ -0,0 +1 @@ +.qa/cephfs/overrides/whitelist_wrongly_marked_down.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/clusters/.qa b/ceph/qa/suites/fs/full/tasks/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/clusters/.qa rename to ceph/qa/suites/fs/full/tasks/.qa diff --git a/ceph/qa/suites/fs/full/tasks/mgr-osd-full.yaml b/ceph/qa/suites/fs/full/tasks/mgr-osd-full.yaml new file mode 100644 index 000000000..88d6527bf --- /dev/null +++ b/ceph/qa/suites/fs/full/tasks/mgr-osd-full.yaml @@ -0,0 +1,21 @@ +overrides: + ceph: + conf: + global: + osd_pool_default_size: 1 + osd_pool_default_min_size: 1 + client: + debug ms: 1 + debug client: 20 + mds: + debug ms: 1 + debug mds: 20 + osd: # force bluestore since it's required for ec overwrites + osd objectstore: bluestore + bluestore block size: 1073741824 +tasks: +- workunit: + cleanup: false + clients: + client.0: + - fs/full/subvolume_rm.sh diff --git a/ceph/qa/suites/fs/mirror/overrides/whitelist_health.yaml b/ceph/qa/suites/fs/mirror/overrides/whitelist_health.yaml deleted file mode 120000 index 3f8d0af87..000000000 --- a/ceph/qa/suites/fs/mirror/overrides/whitelist_health.yaml +++ /dev/null @@ -1 +0,0 @@ -./.qa/cephfs/overrides/whitelist_health.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/mirror/overrides/whitelist_health.yaml b/ceph/qa/suites/fs/mirror/overrides/whitelist_health.yaml new file mode 100644 index 000000000..d40fa4cb8 --- /dev/null +++ b/ceph/qa/suites/fs/mirror/overrides/whitelist_health.yaml @@ -0,0 +1,14 @@ +overrides: + ceph: + log-ignorelist: + - overall HEALTH_ + - \(FS_DEGRADED\) + - \(MDS_FAILED\) + - \(MDS_DEGRADED\) + - \(FS_WITH_FAILED_MDS\) + - \(MDS_DAMAGE\) + - \(MDS_ALL_DOWN\) + - \(MDS_UP_LESS_THAN_MAX\) + - \(FS_INLINE_DATA_DEPRECATED\) + - Reduced data availability + - Degraded data redundancy diff --git a/ceph/qa/suites/fs/shell/tasks/cephfs-shell.yaml b/ceph/qa/suites/fs/shell/tasks/cephfs-shell.yaml index 93c1eb54e..9708252e9 100644 --- a/ceph/qa/suites/fs/shell/tasks/cephfs-shell.yaml +++ b/ceph/qa/suites/fs/shell/tasks/cephfs-shell.yaml @@ -1,7 +1,7 @@ # Right now, cephfs-shell is only available as a package on Ubuntu # This overrides the random distribution that's chosen in the other yaml fragments. os_type: ubuntu -os_version: "18.04" +os_version: "20.04" tasks: - cephfs_test_runner: modules: diff --git a/ceph/qa/suites/fs/upgrade/featureful_client/old_client/centos_latest.yaml b/ceph/qa/suites/fs/upgrade/featureful_client/old_client/centos_latest.yaml new file mode 120000 index 000000000..bd9854e70 --- /dev/null +++ b/ceph/qa/suites/fs/upgrade/featureful_client/old_client/centos_latest.yaml @@ -0,0 +1 @@ +.qa/distros/supported/centos_latest.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/upgrade/featureful_client/old_client/tasks/0-nautilus.yaml b/ceph/qa/suites/fs/upgrade/featureful_client/old_client/tasks/0-nautilus.yaml index c89d18be6..6508bc642 100644 --- a/ceph/qa/suites/fs/upgrade/featureful_client/old_client/tasks/0-nautilus.yaml +++ b/ceph/qa/suites/fs/upgrade/featureful_client/old_client/tasks/0-nautilus.yaml @@ -11,6 +11,11 @@ tasks: - ceph-mgr-rook - ceph-mgr-cephadm - cephadm + - ceph-immutable-object-cache + - python3-rados + - python3-rgw + - python3-rbd + - python3-cephfs extra_packages: ['librados2'] - print: "**** done installing nautilus" - ceph: diff --git a/ceph/qa/suites/fs/upgrade/featureful_client/upgraded_client/centos_latest.yaml b/ceph/qa/suites/fs/upgrade/featureful_client/upgraded_client/centos_latest.yaml new file mode 120000 index 000000000..bd9854e70 --- /dev/null +++ b/ceph/qa/suites/fs/upgrade/featureful_client/upgraded_client/centos_latest.yaml @@ -0,0 +1 @@ +.qa/distros/supported/centos_latest.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/0-nautilus.yaml b/ceph/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/0-nautilus.yaml index c89d18be6..6508bc642 100644 --- a/ceph/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/0-nautilus.yaml +++ b/ceph/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/0-nautilus.yaml @@ -11,6 +11,11 @@ tasks: - ceph-mgr-rook - ceph-mgr-cephadm - cephadm + - ceph-immutable-object-cache + - python3-rados + - python3-rgw + - python3-rbd + - python3-cephfs extra_packages: ['librados2'] - print: "**** done installing nautilus" - ceph: diff --git a/ceph/qa/suites/fs/upgrade/volumes/import-legacy/tasks/0-nautilus.yaml b/ceph/qa/suites/fs/upgrade/volumes/import-legacy/tasks/0-nautilus.yaml index 462163e3b..a7673068f 100644 --- a/ceph/qa/suites/fs/upgrade/volumes/import-legacy/tasks/0-nautilus.yaml +++ b/ceph/qa/suites/fs/upgrade/volumes/import-legacy/tasks/0-nautilus.yaml @@ -11,6 +11,11 @@ tasks: - ceph-mgr-rook - ceph-mgr-cephadm - cephadm + - ceph-immutable-object-cache + - python3-rados + - python3-rgw + - python3-rbd + - python3-cephfs extra_packages: ['librados2'] - print: "**** done installing nautilus" - ceph: diff --git a/ceph/qa/suites/fs/valgrind/mirror/overrides/whitelist_health.yaml b/ceph/qa/suites/fs/valgrind/mirror/overrides/whitelist_health.yaml deleted file mode 120000 index 74f39a49b..000000000 --- a/ceph/qa/suites/fs/valgrind/mirror/overrides/whitelist_health.yaml +++ /dev/null @@ -1 +0,0 @@ -.qa/cephfs/overrides/whitelist_health.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/valgrind/mirror/overrides/whitelist_health.yaml b/ceph/qa/suites/fs/valgrind/mirror/overrides/whitelist_health.yaml new file mode 100644 index 000000000..d40fa4cb8 --- /dev/null +++ b/ceph/qa/suites/fs/valgrind/mirror/overrides/whitelist_health.yaml @@ -0,0 +1,14 @@ +overrides: + ceph: + log-ignorelist: + - overall HEALTH_ + - \(FS_DEGRADED\) + - \(MDS_FAILED\) + - \(MDS_DEGRADED\) + - \(FS_WITH_FAILED_MDS\) + - \(MDS_DAMAGE\) + - \(MDS_ALL_DOWN\) + - \(MDS_UP_LESS_THAN_MAX\) + - \(FS_INLINE_DATA_DEPRECATED\) + - Reduced data availability + - Degraded data redundancy diff --git a/ceph/qa/suites/fs/verify/distro~HEAD b/ceph/qa/suites/fs/verify/distro~HEAD deleted file mode 120000 index 78f2991b4..000000000 --- a/ceph/qa/suites/fs/verify/distro~HEAD +++ /dev/null @@ -1 +0,0 @@ -.qa/distros/supported-random-distro$/ \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/orchestrator_cli/% b/ceph/qa/suites/fs/workload/wsync/$ similarity index 100% rename from ceph/qa/suites/rados/cephadm/orchestrator_cli/% rename to ceph/qa/suites/fs/workload/wsync/$ diff --git a/ceph/qa/suites/fs/workload/wsync/no.yaml b/ceph/qa/suites/fs/workload/wsync/no.yaml new file mode 100644 index 000000000..1ed9e9953 --- /dev/null +++ b/ceph/qa/suites/fs/workload/wsync/no.yaml @@ -0,0 +1,3 @@ +overrides: + kclient: + mntopts: ["nowsync"] diff --git a/ceph/qa/suites/fs/workload/wsync/yes.yaml b/ceph/qa/suites/fs/workload/wsync/yes.yaml new file mode 100644 index 000000000..2061bac11 --- /dev/null +++ b/ceph/qa/suites/fs/workload/wsync/yes.yaml @@ -0,0 +1,3 @@ +overrides: + kclient: + mntopts: ["wsync"] diff --git a/ceph/qa/suites/rados/cephadm/.qa b/ceph/qa/suites/orch/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/.qa rename to ceph/qa/suites/orch/.qa diff --git a/ceph/qa/suites/rados/cephadm/dashboard/.qa b/ceph/qa/suites/orch/cephadm/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/dashboard/.qa rename to ceph/qa/suites/orch/cephadm/.qa diff --git a/ceph/qa/suites/rados/cephadm/smoke-roleless/% b/ceph/qa/suites/orch/cephadm/dashboard/% similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke-roleless/% rename to ceph/qa/suites/orch/cephadm/dashboard/% diff --git a/ceph/qa/suites/rados/cephadm/dashboard/0-distro/.qa b/ceph/qa/suites/orch/cephadm/dashboard/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/dashboard/0-distro/.qa rename to ceph/qa/suites/orch/cephadm/dashboard/.qa diff --git a/ceph/qa/suites/rados/cephadm/orchestrator_cli/.qa b/ceph/qa/suites/orch/cephadm/dashboard/0-distro/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/orchestrator_cli/.qa rename to ceph/qa/suites/orch/cephadm/dashboard/0-distro/.qa diff --git a/ceph/qa/suites/rados/cephadm/dashboard/0-distro/centos_8.2_kubic_stable.yaml b/ceph/qa/suites/orch/cephadm/dashboard/0-distro/centos_8.2_kubic_stable.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/dashboard/0-distro/centos_8.2_kubic_stable.yaml rename to ceph/qa/suites/orch/cephadm/dashboard/0-distro/centos_8.2_kubic_stable.yaml diff --git a/ceph/qa/suites/rados/cephadm/dashboard/task/test_e2e.yaml b/ceph/qa/suites/orch/cephadm/dashboard/task/test_e2e.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/dashboard/task/test_e2e.yaml rename to ceph/qa/suites/orch/cephadm/dashboard/task/test_e2e.yaml diff --git a/ceph/qa/suites/rados/cephadm/smoke/% b/ceph/qa/suites/orch/cephadm/orchestrator_cli/% similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke/% rename to ceph/qa/suites/orch/cephadm/orchestrator_cli/% diff --git a/ceph/qa/suites/rados/cephadm/smoke-roleless/.qa b/ceph/qa/suites/orch/cephadm/orchestrator_cli/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke-roleless/.qa rename to ceph/qa/suites/orch/cephadm/orchestrator_cli/.qa diff --git a/ceph/qa/suites/rados/cephadm/orchestrator_cli/0-random-distro$ b/ceph/qa/suites/orch/cephadm/orchestrator_cli/0-random-distro$ similarity index 100% rename from ceph/qa/suites/rados/cephadm/orchestrator_cli/0-random-distro$ rename to ceph/qa/suites/orch/cephadm/orchestrator_cli/0-random-distro$ diff --git a/ceph/qa/suites/rados/cephadm/orchestrator_cli/2-node-mgr.yaml b/ceph/qa/suites/orch/cephadm/orchestrator_cli/2-node-mgr.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/orchestrator_cli/2-node-mgr.yaml rename to ceph/qa/suites/orch/cephadm/orchestrator_cli/2-node-mgr.yaml diff --git a/ceph/qa/suites/rados/cephadm/orchestrator_cli/orchestrator_cli.yaml b/ceph/qa/suites/orch/cephadm/orchestrator_cli/orchestrator_cli.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/orchestrator_cli/orchestrator_cli.yaml rename to ceph/qa/suites/orch/cephadm/orchestrator_cli/orchestrator_cli.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/% b/ceph/qa/suites/orch/cephadm/smoke-roleless/% similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/% rename to ceph/qa/suites/orch/cephadm/smoke-roleless/% diff --git a/ceph/qa/suites/rados/cephadm/smoke/.qa b/ceph/qa/suites/orch/cephadm/smoke-roleless/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke/.qa rename to ceph/qa/suites/orch/cephadm/smoke-roleless/.qa diff --git a/ceph/qa/suites/rados/cephadm/smoke-roleless/0-distro b/ceph/qa/suites/orch/cephadm/smoke-roleless/0-distro similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke-roleless/0-distro rename to ceph/qa/suites/orch/cephadm/smoke-roleless/0-distro diff --git a/ceph/qa/suites/rados/cephadm/smoke-roleless/1-start.yaml b/ceph/qa/suites/orch/cephadm/smoke-roleless/1-start.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke-roleless/1-start.yaml rename to ceph/qa/suites/orch/cephadm/smoke-roleless/1-start.yaml diff --git a/ceph/qa/suites/rados/cephadm/smoke-roleless/2-services/basic.yaml b/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/basic.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke-roleless/2-services/basic.yaml rename to ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/basic.yaml diff --git a/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/client-keyring.yaml b/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/client-keyring.yaml new file mode 100644 index 000000000..f00800471 --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/client-keyring.yaml @@ -0,0 +1,40 @@ +tasks: +- cephadm.shell: + host.a: + - ceph orch host label add `hostname` foo + - ceph auth get-or-create client.foo mon 'allow r' + - ceph orch client-keyring set client.foo label:foo --mode 770 --owner 11111:22222 +- exec: + host.a: + - while ! test -e /etc/ceph/ceph.client.foo.keyring ; do sleep 1 ; done + - ls -al /etc/ceph/ceph.client.foo.keyring | grep rwxrwx--- + - ls -al /etc/ceph/ceph.client.foo.keyring | grep 11111 + - ls -al /etc/ceph/ceph.client.foo.keyring | grep 22222 + - test -e /etc/ceph/ceph.conf +- exec: + host.b: + - test ! -e /etc/ceph/ceph.client.foo.keyring +- cephadm.shell: + host.b: + - ceph orch host label add `hostname` foo +- exec: + host.b: + - while ! test -e /etc/ceph/ceph.client.foo.keyring ; do sleep 1 ; done + - ls -al /etc/ceph/ceph.client.foo.keyring | grep rwxrwx--- + - ls -al /etc/ceph/ceph.client.foo.keyring | grep 11111 + - ls -al /etc/ceph/ceph.client.foo.keyring | grep 22222 +- cephadm.shell: + host.b: + - ceph orch host label rm `hostname` foo +- exec: + host.b: + - while test -e /etc/ceph/ceph.client.foo.keyring ; do sleep 1 ; done +- exec: + host.a: + - test -e /etc/ceph/ceph.client.foo.keyring +- cephadm.shell: + host.a: + - ceph orch client-keyring rm client.foo +- exec: + host.a: + - while test -e /etc/ceph/ceph.client.foo.keyring ; do sleep 1 ; done diff --git a/ceph/qa/suites/rados/cephadm/smoke-roleless/2-services/iscsi.yaml b/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/iscsi.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke-roleless/2-services/iscsi.yaml rename to ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/iscsi.yaml diff --git a/ceph/qa/suites/rados/cephadm/smoke-roleless/2-services/mirror.yaml b/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/mirror.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke-roleless/2-services/mirror.yaml rename to ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/mirror.yaml diff --git a/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress.yaml b/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress.yaml new file mode 100644 index 000000000..3e5ad1a2e --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress.yaml @@ -0,0 +1,68 @@ +tasks: +- vip: + +# make sure cephadm notices the new IP +- cephadm.shell: + host.a: + - ceph orch device ls --refresh + +# stop kernel nfs server, if running +- vip.exec: + all-hosts: + - systemctl stop nfs-server + +- cephadm.shell: + host.a: + - ceph fs volume create foofs + +# deploy nfs + ingress +- cephadm.apply: + specs: + - service_type: nfs + service_id: foo + placement: + count: 2 + spec: + port: 12049 + - service_type: ingress + service_id: nfs.foo + spec: + backend_service: nfs.foo + frontend_port: 2049 + monitor_port: 9002 + virtual_ip: "{{VIP0}}/{{VIPPREFIXLEN}}" +- cephadm.wait_for_service: + service: nfs.foo +- cephadm.wait_for_service: + service: ingress.nfs.foo + +## export and mount + +- cephadm.shell: + host.a: + - ceph nfs export create cephfs foofs foo --binding /fake + +- vip.exec: + host.a: + - mkdir /mnt/foo + - sleep 5 + - mount -t nfs {{VIP0}}:/fake /mnt/foo + - echo test > /mnt/foo/testfile + - sync + +# take each gateway down in turn and ensure things still work +- cephadm.shell: + volumes: + - /mnt/foo:/mnt/foo + host.a: + - | + echo "Check with each haproxy down in turn..." + for haproxy in `ceph orch ps | grep ^haproxy.nfs.foo. | awk '{print $1}'`; do + ceph orch daemon stop $haproxy + while ! ceph orch ps | grep $haproxy | grep stopped; do sleep 1 ; done + cat /mnt/foo/testfile + echo $haproxy > /mnt/foo/testfile + sync + ceph orch daemon start $haproxy + while ! ceph orch ps | grep $haproxy | grep running; do sleep 1 ; done + done diff --git a/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress2.yaml b/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress2.yaml new file mode 100644 index 000000000..09fb3c768 --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress2.yaml @@ -0,0 +1,50 @@ +tasks: +- vip: + +# make sure cephadm notices the new IP +- cephadm.shell: + host.a: + - ceph orch device ls --refresh + +# stop kernel nfs server, if running +- vip.exec: + all-hosts: + - systemctl stop nfs-server + +- cephadm.shell: + host.a: + - ceph fs volume create foofs + - ceph nfs cluster create foo --ingress --virtual-ip {{VIP0}}/{{VIPPREFIXLEN}} + - ceph nfs export create cephfs foofs foo --binding /fake + +- cephadm.wait_for_service: + service: nfs.foo +- cephadm.wait_for_service: + service: ingress.nfs.foo + +## export and mount + +- vip.exec: + host.a: + - mkdir /mnt/foo + - sleep 5 + - mount -t nfs {{VIP0}}:/fake /mnt/foo + - echo test > /mnt/foo/testfile + - sync + +# take each gateway down in turn and ensure things still work +- cephadm.shell: + volumes: + - /mnt/foo:/mnt/foo + host.a: + - | + echo "Check with each haproxy down in turn..." + for haproxy in `ceph orch ps | grep ^haproxy.nfs.foo. | awk '{print $1}'`; do + ceph orch daemon stop $haproxy + while ! ceph orch ps | grep $haproxy | grep stopped; do sleep 1 ; done + cat /mnt/foo/testfile + echo $haproxy > /mnt/foo/testfile + sync + ceph orch daemon start $haproxy + while ! ceph orch ps | grep $haproxy | grep running; do sleep 1 ; done + done diff --git a/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs.yaml b/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs.yaml new file mode 100644 index 000000000..194f4e9de --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs.yaml @@ -0,0 +1,13 @@ +tasks: + +# stop kernel nfs server, if running +- vip.exec: + all-hosts: + - systemctl stop nfs-server + +- cephadm.apply: + specs: + - service_type: nfs + service_id: foo +- cephadm.wait_for_service: + service: nfs.foo diff --git a/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs2.yaml b/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs2.yaml new file mode 100644 index 000000000..959c5aa77 --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs2.yaml @@ -0,0 +1,12 @@ +tasks: + +# stop kernel nfs server, if running +- vip.exec: + all-hosts: + - systemctl stop nfs-server + +- cephadm.shell: + host.a: + - ceph nfs cluster create foo +- cephadm.wait_for_service: + service: nfs.foo diff --git a/ceph/qa/suites/rados/cephadm/smoke-roleless/2-services/rgw-ingress.yaml b/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/rgw-ingress.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke-roleless/2-services/rgw-ingress.yaml rename to ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/rgw-ingress.yaml diff --git a/ceph/qa/suites/rados/cephadm/smoke-roleless/2-services/rgw.yaml b/ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/rgw.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke-roleless/2-services/rgw.yaml rename to ceph/qa/suites/orch/cephadm/smoke-roleless/2-services/rgw.yaml diff --git a/ceph/qa/suites/rados/cephadm/smoke-roleless/3-final.yaml b/ceph/qa/suites/orch/cephadm/smoke-roleless/3-final.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke-roleless/3-final.yaml rename to ceph/qa/suites/orch/cephadm/smoke-roleless/3-final.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash/% b/ceph/qa/suites/orch/cephadm/smoke-singlehost/% similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash/% rename to ceph/qa/suites/orch/cephadm/smoke-singlehost/% diff --git a/ceph/qa/suites/orch/cephadm/smoke-singlehost/0-distro$ b/ceph/qa/suites/orch/cephadm/smoke-singlehost/0-distro$ new file mode 120000 index 000000000..d2dffb181 --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/smoke-singlehost/0-distro$ @@ -0,0 +1 @@ +../smoke/distro \ No newline at end of file diff --git a/ceph/qa/suites/orch/cephadm/smoke-singlehost/1-start.yaml b/ceph/qa/suites/orch/cephadm/smoke-singlehost/1-start.yaml new file mode 100644 index 000000000..ca6019c66 --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/smoke-singlehost/1-start.yaml @@ -0,0 +1,27 @@ +tasks: +- cephadm: + roleless: true + single_host_defaults: true +- cephadm.shell: + host.a: + - ceph orch status + - ceph orch ps + - ceph orch ls + - ceph orch host ls + - ceph orch device ls +roles: +- - host.a + - osd.0 + - osd.1 + - osd.2 + - osd.3 + - client.0 +openstack: +- volumes: # attached to each instance + count: 4 + size: 10 # GB +overrides: + ceph: + conf: + osd: + osd shutdown pgref assert: true diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/backoff/normal.yaml b/ceph/qa/suites/orch/cephadm/smoke-singlehost/2-services/basic.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/backoff/normal.yaml rename to ceph/qa/suites/orch/cephadm/smoke-singlehost/2-services/basic.yaml diff --git a/ceph/qa/suites/orch/cephadm/smoke-singlehost/2-services/rgw.yaml b/ceph/qa/suites/orch/cephadm/smoke-singlehost/2-services/rgw.yaml new file mode 100644 index 000000000..cb2c6f4b6 --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/smoke-singlehost/2-services/rgw.yaml @@ -0,0 +1,12 @@ +tasks: +- cephadm.apply: + specs: + - service_type: rgw + service_id: foo + placement: + count_per_host: 4 + host_pattern: "*" + spec: + rgw_frontend_port: 8000 +- cephadm.wait_for_service: + service: rgw.foo diff --git a/ceph/qa/suites/orch/cephadm/smoke-singlehost/3-final.yaml b/ceph/qa/suites/orch/cephadm/smoke-singlehost/3-final.yaml new file mode 100644 index 000000000..02f5b289c --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/smoke-singlehost/3-final.yaml @@ -0,0 +1,8 @@ +tasks: +- cephadm.shell: + host.a: + - ceph orch status + - ceph orch ps + - ceph orch ls + - ceph orch host ls + - ceph orch device ls diff --git a/ceph/qa/suites/rados/cephadm/upgrade/% b/ceph/qa/suites/orch/cephadm/smoke/% similarity index 100% rename from ceph/qa/suites/rados/cephadm/upgrade/% rename to ceph/qa/suites/orch/cephadm/smoke/% diff --git a/ceph/qa/suites/rados/cephadm/smoke/distro/.qa b/ceph/qa/suites/orch/cephadm/smoke/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke/distro/.qa rename to ceph/qa/suites/orch/cephadm/smoke/.qa diff --git a/ceph/qa/suites/rados/cephadm/thrash/.qa b/ceph/qa/suites/orch/cephadm/smoke/distro/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash/.qa rename to ceph/qa/suites/orch/cephadm/smoke/distro/.qa diff --git a/ceph/qa/suites/rados/cephadm/smoke/distro/centos_8.2_kubic_stable.yaml b/ceph/qa/suites/orch/cephadm/smoke/distro/centos_8.2_kubic_stable.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke/distro/centos_8.2_kubic_stable.yaml rename to ceph/qa/suites/orch/cephadm/smoke/distro/centos_8.2_kubic_stable.yaml diff --git a/ceph/qa/suites/rados/cephadm/smoke/distro/rhel_8.3_kubic_stable.yaml b/ceph/qa/suites/orch/cephadm/smoke/distro/rhel_8.3_kubic_stable.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke/distro/rhel_8.3_kubic_stable.yaml rename to ceph/qa/suites/orch/cephadm/smoke/distro/rhel_8.3_kubic_stable.yaml diff --git a/ceph/qa/suites/rados/cephadm/smoke/distro/ubuntu_18.04.yaml b/ceph/qa/suites/orch/cephadm/smoke/distro/ubuntu_18.04.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke/distro/ubuntu_18.04.yaml rename to ceph/qa/suites/orch/cephadm/smoke/distro/ubuntu_18.04.yaml diff --git a/ceph/qa/suites/rados/cephadm/smoke/distro/ubuntu_20.04.yaml b/ceph/qa/suites/orch/cephadm/smoke/distro/ubuntu_20.04.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke/distro/ubuntu_20.04.yaml rename to ceph/qa/suites/orch/cephadm/smoke/distro/ubuntu_20.04.yaml diff --git a/ceph/qa/suites/rados/cephadm/smoke/fixed-2.yaml b/ceph/qa/suites/orch/cephadm/smoke/fixed-2.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke/fixed-2.yaml rename to ceph/qa/suites/orch/cephadm/smoke/fixed-2.yaml diff --git a/ceph/qa/suites/rados/cephadm/smoke/mon_election b/ceph/qa/suites/orch/cephadm/smoke/mon_election similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke/mon_election rename to ceph/qa/suites/orch/cephadm/smoke/mon_election diff --git a/ceph/qa/suites/rados/cephadm/smoke/start.yaml b/ceph/qa/suites/orch/cephadm/smoke/start.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/smoke/start.yaml rename to ceph/qa/suites/orch/cephadm/smoke/start.yaml diff --git a/ceph/qa/suites/rados/cephadm/with-work/% b/ceph/qa/suites/orch/cephadm/thrash-old-clients/% similarity index 100% rename from ceph/qa/suites/rados/cephadm/with-work/% rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/% diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/d-balancer/.qa b/ceph/qa/suites/orch/cephadm/thrash-old-clients/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/d-balancer/.qa rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/.qa diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/distro$/.qa b/ceph/qa/suites/orch/cephadm/thrash-old-clients/0-size-min-size-overrides/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/distro$/.qa rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/0-size-min-size-overrides/.qa diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/0-size-min-size-overrides/2-size-2-min-size.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/0-size-min-size-overrides/2-size-2-min-size.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/0-size-min-size-overrides/2-size-2-min-size.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/0-size-min-size-overrides/2-size-2-min-size.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/0-size-min-size-overrides/3-size-2-min-size.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/0-size-min-size-overrides/3-size-2-min-size.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/0-size-min-size-overrides/3-size-2-min-size.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/0-size-min-size-overrides/3-size-2-min-size.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/msgr-failures/.qa b/ceph/qa/suites/orch/cephadm/thrash-old-clients/1-install/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/msgr-failures/.qa rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/1-install/.qa diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/1-install/luminous-v1only.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/1-install/luminous-v1only.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/1-install/luminous-v1only.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/1-install/luminous-v1only.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/1-install/luminous.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/1-install/luminous.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/1-install/luminous.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/1-install/luminous.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/1-install/mimic-v1only.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/1-install/mimic-v1only.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/1-install/mimic-v1only.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/1-install/mimic-v1only.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/1-install/mimic.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/1-install/mimic.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/1-install/mimic.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/1-install/mimic.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/1-install/nautilus-v1only.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/1-install/nautilus-v1only.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/1-install/nautilus-v1only.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/1-install/nautilus-v1only.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/1-install/nautilus-v2only.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/1-install/nautilus-v2only.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/1-install/nautilus-v2only.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/1-install/nautilus-v2only.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/1-install/nautilus.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/1-install/nautilus.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/1-install/nautilus.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/1-install/nautilus.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/1-install/octopus.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/1-install/octopus.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/1-install/octopus.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/1-install/octopus.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/thrashers/.qa b/ceph/qa/suites/orch/cephadm/thrash-old-clients/backoff/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/thrashers/.qa rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/backoff/.qa diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/d-balancer/on.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/backoff/normal.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/d-balancer/on.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/backoff/normal.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/backoff/peering.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/backoff/peering.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/backoff/peering.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/backoff/peering.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/backoff/peering_and_degraded.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/backoff/peering_and_degraded.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/backoff/peering_and_degraded.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/backoff/peering_and_degraded.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/ceph.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/ceph.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/ceph.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/ceph.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/thrashers/none.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/clusters/+ similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/thrashers/none.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/clusters/+ diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/workloads/.qa b/ceph/qa/suites/orch/cephadm/thrash-old-clients/clusters/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/workloads/.qa rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/clusters/.qa diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/clusters/openstack.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/clusters/openstack.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/clusters/openstack.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/clusters/openstack.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/clusters/three-plus-one.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/clusters/three-plus-one.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/clusters/three-plus-one.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/clusters/three-plus-one.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash/3-tasks/.qa b/ceph/qa/suites/orch/cephadm/thrash-old-clients/d-balancer/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash/3-tasks/.qa rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/d-balancer/.qa diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/d-balancer/crush-compat.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/d-balancer/crush-compat.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/d-balancer/crush-compat.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/d-balancer/crush-compat.yaml diff --git a/ceph/qa/suites/rados/cephadm/upgrade/2-repo_digest/defaut.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/d-balancer/on.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/upgrade/2-repo_digest/defaut.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/d-balancer/on.yaml diff --git a/ceph/qa/suites/rados/cephadm/with-work/.qa b/ceph/qa/suites/orch/cephadm/thrash-old-clients/distro$/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/with-work/.qa rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/distro$/.qa diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/distro$/ubuntu_18.04.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/distro$/ubuntu_18.04.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/distro$/ubuntu_18.04.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/distro$/ubuntu_18.04.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/mon_election b/ceph/qa/suites/orch/cephadm/thrash-old-clients/mon_election similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/mon_election rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/mon_election diff --git a/ceph/qa/suites/rados/cephadm/with-work/mode/.qa b/ceph/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/with-work/mode/.qa rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/.qa diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/msgr-failures/fastclose.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/fastclose.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/msgr-failures/fastclose.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/fastclose.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/msgr-failures/few.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/few.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/msgr-failures/few.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/few.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/msgr-failures/osd-delay.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/osd-delay.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/msgr-failures/osd-delay.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/osd-delay.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/rados.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/rados.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/rados.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/rados.yaml diff --git a/ceph/qa/suites/rados/cephadm/with-work/tasks/.qa b/ceph/qa/suites/orch/cephadm/thrash-old-clients/thrashers/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/with-work/tasks/.qa rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/thrashers/.qa diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/thrashers/careful.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/thrashers/careful.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/thrashers/careful.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/thrashers/careful.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/thrashers/default.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/thrashers/default.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/thrashers/default.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/thrashers/default.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/thrashers/mapgap.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/thrashers/mapgap.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/thrashers/mapgap.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/thrashers/mapgap.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/thrashers/morepggrow.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/thrashers/morepggrow.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/thrashers/morepggrow.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/thrashers/morepggrow.yaml diff --git a/ceph/qa/suites/rados/cephadm/workunits/% b/ceph/qa/suites/orch/cephadm/thrash-old-clients/thrashers/none.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/workunits/% rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/thrashers/none.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/thrashers/pggrow.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/thrashers/pggrow.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/thrashers/pggrow.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/thrashers/pggrow.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/thrashosds-health.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/thrashosds-health.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/thrashosds-health.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/thrashosds-health.yaml diff --git a/ceph/qa/suites/orch/cephadm/thrash-old-clients/workloads/.qa b/ceph/qa/suites/orch/cephadm/thrash-old-clients/workloads/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/thrash-old-clients/workloads/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/workloads/cache-snaps.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/workloads/cache-snaps.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/workloads/cache-snaps.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/workloads/cache-snaps.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/workloads/radosbench.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/workloads/radosbench.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/workloads/radosbench.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/workloads/radosbench.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/workloads/rbd_cls.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/workloads/rbd_cls.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/workloads/rbd_cls.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/workloads/rbd_cls.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/workloads/snaps-few-objects.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/workloads/snaps-few-objects.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/workloads/snaps-few-objects.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/workloads/snaps-few-objects.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash-old-clients/workloads/test_rbd_api.yaml b/ceph/qa/suites/orch/cephadm/thrash-old-clients/workloads/test_rbd_api.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash-old-clients/workloads/test_rbd_api.yaml rename to ceph/qa/suites/orch/cephadm/thrash-old-clients/workloads/test_rbd_api.yaml diff --git a/ceph/qa/suites/orch/cephadm/thrash/% b/ceph/qa/suites/orch/cephadm/thrash/% new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/rados/cephadm/upgrade/.qa b/ceph/qa/suites/orch/cephadm/thrash/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/upgrade/.qa rename to ceph/qa/suites/orch/cephadm/thrash/.qa diff --git a/ceph/qa/suites/rados/cephadm/thrash/0-distro b/ceph/qa/suites/orch/cephadm/thrash/0-distro similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash/0-distro rename to ceph/qa/suites/orch/cephadm/thrash/0-distro diff --git a/ceph/qa/suites/rados/cephadm/thrash/1-start.yaml b/ceph/qa/suites/orch/cephadm/thrash/1-start.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash/1-start.yaml rename to ceph/qa/suites/orch/cephadm/thrash/1-start.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash/2-thrash.yaml b/ceph/qa/suites/orch/cephadm/thrash/2-thrash.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash/2-thrash.yaml rename to ceph/qa/suites/orch/cephadm/thrash/2-thrash.yaml diff --git a/ceph/qa/suites/orch/cephadm/thrash/3-tasks/.qa b/ceph/qa/suites/orch/cephadm/thrash/3-tasks/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/thrash/3-tasks/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/orch/cephadm/thrash/3-tasks/rados_api_tests.yaml b/ceph/qa/suites/orch/cephadm/thrash/3-tasks/rados_api_tests.yaml new file mode 120000 index 000000000..34e657e04 --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/thrash/3-tasks/rados_api_tests.yaml @@ -0,0 +1 @@ +.qa/suites/rados/thrash/workloads/rados_api_tests.yaml \ No newline at end of file diff --git a/ceph/qa/suites/orch/cephadm/thrash/3-tasks/radosbench.yaml b/ceph/qa/suites/orch/cephadm/thrash/3-tasks/radosbench.yaml new file mode 120000 index 000000000..dad17e0de --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/thrash/3-tasks/radosbench.yaml @@ -0,0 +1 @@ +.qa/suites/rados/thrash/workloads/radosbench.yaml \ No newline at end of file diff --git a/ceph/qa/suites/orch/cephadm/thrash/3-tasks/small-objects.yaml b/ceph/qa/suites/orch/cephadm/thrash/3-tasks/small-objects.yaml new file mode 120000 index 000000000..6aa66aa37 --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/thrash/3-tasks/small-objects.yaml @@ -0,0 +1 @@ +.qa/suites/rados/thrash/workloads/small-objects.yaml \ No newline at end of file diff --git a/ceph/qa/suites/orch/cephadm/thrash/3-tasks/snaps-few-objects.yaml b/ceph/qa/suites/orch/cephadm/thrash/3-tasks/snaps-few-objects.yaml new file mode 120000 index 000000000..c9cc4cd3e --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/thrash/3-tasks/snaps-few-objects.yaml @@ -0,0 +1 @@ +.qa/suites/rados/thrash/workloads/snaps-few-objects.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/thrash/fixed-2.yaml b/ceph/qa/suites/orch/cephadm/thrash/fixed-2.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash/fixed-2.yaml rename to ceph/qa/suites/orch/cephadm/thrash/fixed-2.yaml diff --git a/ceph/qa/suites/rados/cephadm/thrash/msgr b/ceph/qa/suites/orch/cephadm/thrash/msgr similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash/msgr rename to ceph/qa/suites/orch/cephadm/thrash/msgr diff --git a/ceph/qa/suites/rados/cephadm/thrash/root.yaml b/ceph/qa/suites/orch/cephadm/thrash/root.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/thrash/root.yaml rename to ceph/qa/suites/orch/cephadm/thrash/root.yaml diff --git a/ceph/qa/suites/orch/cephadm/upgrade/% b/ceph/qa/suites/orch/cephadm/upgrade/% new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/rados/cephadm/workunits/.qa b/ceph/qa/suites/orch/cephadm/upgrade/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/workunits/.qa rename to ceph/qa/suites/orch/cephadm/upgrade/.qa diff --git a/ceph/qa/suites/rados/cephadm/upgrade/1-start-distro/1-start-centos_8.3-octopus.yaml b/ceph/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_8.3-octopus.yaml similarity index 95% rename from ceph/qa/suites/rados/cephadm/upgrade/1-start-distro/1-start-centos_8.3-octopus.yaml rename to ceph/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_8.3-octopus.yaml index f5441eb93..2e551dd43 100644 --- a/ceph/qa/suites/rados/cephadm/upgrade/1-start-distro/1-start-centos_8.3-octopus.yaml +++ b/ceph/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_8.3-octopus.yaml @@ -14,6 +14,7 @@ tasks: allow_ptrace: false # deploy additional mons the "old" (octopus) way add_mons_via_daemon_add: true + avoid_pacific_features: true roles: diff --git a/ceph/qa/suites/rados/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04-15.2.9.yaml b/ceph/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04-15.2.9.yaml similarity index 95% rename from ceph/qa/suites/rados/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04-15.2.9.yaml rename to ceph/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04-15.2.9.yaml index e60b8872d..fde68b32a 100644 --- a/ceph/qa/suites/rados/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04-15.2.9.yaml +++ b/ceph/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04-15.2.9.yaml @@ -10,6 +10,7 @@ tasks: allow_ptrace: false # deploy additional mons the "old" (octopus) way add_mons_via_daemon_add: true + avoid_pacific_features: true roles: - - mon.a diff --git a/ceph/qa/suites/rados/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml b/ceph/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml similarity index 95% rename from ceph/qa/suites/rados/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml rename to ceph/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml index 2af315930..983b4f3dc 100644 --- a/ceph/qa/suites/rados/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml +++ b/ceph/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml @@ -10,6 +10,7 @@ tasks: allow_ptrace: false # deploy additional mons the "old" (octopus) way add_mons_via_daemon_add: true + avoid_pacific_features: true roles: - - mon.a diff --git a/ceph/qa/suites/orch/cephadm/upgrade/2-repo_digest/defaut.yaml b/ceph/qa/suites/orch/cephadm/upgrade/2-repo_digest/defaut.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/rados/cephadm/upgrade/2-repo_digest/repo_digest.yaml b/ceph/qa/suites/orch/cephadm/upgrade/2-repo_digest/repo_digest.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/upgrade/2-repo_digest/repo_digest.yaml rename to ceph/qa/suites/orch/cephadm/upgrade/2-repo_digest/repo_digest.yaml diff --git a/ceph/qa/suites/rados/cephadm/upgrade/3-start-upgrade.yaml b/ceph/qa/suites/orch/cephadm/upgrade/3-start-upgrade.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/upgrade/3-start-upgrade.yaml rename to ceph/qa/suites/orch/cephadm/upgrade/3-start-upgrade.yaml diff --git a/ceph/qa/suites/rados/cephadm/upgrade/4-wait.yaml b/ceph/qa/suites/orch/cephadm/upgrade/4-wait.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/upgrade/4-wait.yaml rename to ceph/qa/suites/orch/cephadm/upgrade/4-wait.yaml diff --git a/ceph/qa/suites/rados/cephadm/upgrade/mon_election b/ceph/qa/suites/orch/cephadm/upgrade/mon_election similarity index 100% rename from ceph/qa/suites/rados/cephadm/upgrade/mon_election rename to ceph/qa/suites/orch/cephadm/upgrade/mon_election diff --git a/ceph/qa/suites/orch/cephadm/with-work/% b/ceph/qa/suites/orch/cephadm/with-work/% new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/orch/cephadm/with-work/.qa b/ceph/qa/suites/orch/cephadm/with-work/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/with-work/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/with-work/0-distro b/ceph/qa/suites/orch/cephadm/with-work/0-distro similarity index 100% rename from ceph/qa/suites/rados/cephadm/with-work/0-distro rename to ceph/qa/suites/orch/cephadm/with-work/0-distro diff --git a/ceph/qa/suites/rados/cephadm/with-work/fixed-2.yaml b/ceph/qa/suites/orch/cephadm/with-work/fixed-2.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/with-work/fixed-2.yaml rename to ceph/qa/suites/orch/cephadm/with-work/fixed-2.yaml diff --git a/ceph/qa/suites/orch/cephadm/with-work/mode/.qa b/ceph/qa/suites/orch/cephadm/with-work/mode/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/with-work/mode/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/with-work/mode/packaged.yaml b/ceph/qa/suites/orch/cephadm/with-work/mode/packaged.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/with-work/mode/packaged.yaml rename to ceph/qa/suites/orch/cephadm/with-work/mode/packaged.yaml diff --git a/ceph/qa/suites/rados/cephadm/with-work/mode/root.yaml b/ceph/qa/suites/orch/cephadm/with-work/mode/root.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/with-work/mode/root.yaml rename to ceph/qa/suites/orch/cephadm/with-work/mode/root.yaml diff --git a/ceph/qa/suites/rados/cephadm/with-work/mon_election b/ceph/qa/suites/orch/cephadm/with-work/mon_election similarity index 100% rename from ceph/qa/suites/rados/cephadm/with-work/mon_election rename to ceph/qa/suites/orch/cephadm/with-work/mon_election diff --git a/ceph/qa/suites/rados/cephadm/with-work/msgr b/ceph/qa/suites/orch/cephadm/with-work/msgr similarity index 100% rename from ceph/qa/suites/rados/cephadm/with-work/msgr rename to ceph/qa/suites/orch/cephadm/with-work/msgr diff --git a/ceph/qa/suites/rados/cephadm/with-work/start.yaml b/ceph/qa/suites/orch/cephadm/with-work/start.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/with-work/start.yaml rename to ceph/qa/suites/orch/cephadm/with-work/start.yaml diff --git a/ceph/qa/suites/orch/cephadm/with-work/tasks/.qa b/ceph/qa/suites/orch/cephadm/with-work/tasks/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/with-work/tasks/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/orch/cephadm/with-work/tasks/rados_api_tests.yaml b/ceph/qa/suites/orch/cephadm/with-work/tasks/rados_api_tests.yaml new file mode 120000 index 000000000..2ce80f969 --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/with-work/tasks/rados_api_tests.yaml @@ -0,0 +1 @@ +.qa/suites/rados/basic/tasks/rados_api_tests.yaml \ No newline at end of file diff --git a/ceph/qa/suites/orch/cephadm/with-work/tasks/rados_python.yaml b/ceph/qa/suites/orch/cephadm/with-work/tasks/rados_python.yaml new file mode 120000 index 000000000..210ad8f18 --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/with-work/tasks/rados_python.yaml @@ -0,0 +1 @@ +.qa/suites/rados/basic/tasks/rados_python.yaml \ No newline at end of file diff --git a/ceph/qa/suites/orch/cephadm/workunits/% b/ceph/qa/suites/orch/cephadm/workunits/% new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/rados/cephadm/workunits/0-distro/.qa b/ceph/qa/suites/orch/cephadm/workunits/.qa similarity index 100% rename from ceph/qa/suites/rados/cephadm/workunits/0-distro/.qa rename to ceph/qa/suites/orch/cephadm/workunits/.qa diff --git a/ceph/qa/suites/orch/cephadm/workunits/0-distro/.qa b/ceph/qa/suites/orch/cephadm/workunits/0-distro/.qa new file mode 120000 index 000000000..fea2489fd --- /dev/null +++ b/ceph/qa/suites/orch/cephadm/workunits/0-distro/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/workunits/0-distro/centos_8.2_kubic_stable.yaml b/ceph/qa/suites/orch/cephadm/workunits/0-distro/centos_8.2_kubic_stable.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/workunits/0-distro/centos_8.2_kubic_stable.yaml rename to ceph/qa/suites/orch/cephadm/workunits/0-distro/centos_8.2_kubic_stable.yaml diff --git a/ceph/qa/suites/rados/cephadm/workunits/mon_election b/ceph/qa/suites/orch/cephadm/workunits/mon_election similarity index 100% rename from ceph/qa/suites/rados/cephadm/workunits/mon_election rename to ceph/qa/suites/orch/cephadm/workunits/mon_election diff --git a/ceph/qa/suites/rados/cephadm/workunits/task/test_adoption.yaml b/ceph/qa/suites/orch/cephadm/workunits/task/test_adoption.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/workunits/task/test_adoption.yaml rename to ceph/qa/suites/orch/cephadm/workunits/task/test_adoption.yaml diff --git a/ceph/qa/suites/rados/cephadm/workunits/task/test_cephadm.yaml b/ceph/qa/suites/orch/cephadm/workunits/task/test_cephadm.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/workunits/task/test_cephadm.yaml rename to ceph/qa/suites/orch/cephadm/workunits/task/test_cephadm.yaml diff --git a/ceph/qa/suites/rados/cephadm/workunits/task/test_cephadm_repos.yaml b/ceph/qa/suites/orch/cephadm/workunits/task/test_cephadm_repos.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/workunits/task/test_cephadm_repos.yaml rename to ceph/qa/suites/orch/cephadm/workunits/task/test_cephadm_repos.yaml diff --git a/ceph/qa/suites/rados/cephadm/workunits/task/test_orch_cli.yaml b/ceph/qa/suites/orch/cephadm/workunits/task/test_orch_cli.yaml similarity index 100% rename from ceph/qa/suites/rados/cephadm/workunits/task/test_orch_cli.yaml rename to ceph/qa/suites/orch/cephadm/workunits/task/test_orch_cli.yaml diff --git a/ceph/qa/suites/orch/rook/.qa b/ceph/qa/suites/orch/rook/.qa new file mode 120000 index 000000000..fea2489fd --- /dev/null +++ b/ceph/qa/suites/orch/rook/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/ceph/qa/suites/orch/rook/smoke/% b/ceph/qa/suites/orch/rook/smoke/% new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/orch/rook/smoke/.qa b/ceph/qa/suites/orch/rook/smoke/.qa new file mode 120000 index 000000000..fea2489fd --- /dev/null +++ b/ceph/qa/suites/orch/rook/smoke/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/ceph/qa/suites/orch/rook/smoke/0-distro/.qa b/ceph/qa/suites/orch/rook/smoke/0-distro/.qa new file mode 120000 index 000000000..fea2489fd --- /dev/null +++ b/ceph/qa/suites/orch/rook/smoke/0-distro/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/ceph/qa/suites/orch/rook/smoke/0-distro/ubuntu_18.04.yaml b/ceph/qa/suites/orch/rook/smoke/0-distro/ubuntu_18.04.yaml new file mode 100644 index 000000000..4d4464884 --- /dev/null +++ b/ceph/qa/suites/orch/rook/smoke/0-distro/ubuntu_18.04.yaml @@ -0,0 +1,2 @@ +os_type: ubuntu +os_version: "18.04" diff --git a/ceph/qa/suites/orch/rook/smoke/0-distro/ubuntu_20.04.yaml b/ceph/qa/suites/orch/rook/smoke/0-distro/ubuntu_20.04.yaml new file mode 100644 index 000000000..f20398230 --- /dev/null +++ b/ceph/qa/suites/orch/rook/smoke/0-distro/ubuntu_20.04.yaml @@ -0,0 +1,2 @@ +os_type: ubuntu +os_version: "20.04" diff --git a/ceph/qa/suites/orch/rook/smoke/0-kubeadm.yaml b/ceph/qa/suites/orch/rook/smoke/0-kubeadm.yaml new file mode 100644 index 000000000..33915f571 --- /dev/null +++ b/ceph/qa/suites/orch/rook/smoke/0-kubeadm.yaml @@ -0,0 +1,2 @@ +tasks: +- kubeadm: diff --git a/ceph/qa/suites/orch/rook/smoke/1-rook.yaml b/ceph/qa/suites/orch/rook/smoke/1-rook.yaml new file mode 100644 index 000000000..eca74f160 --- /dev/null +++ b/ceph/qa/suites/orch/rook/smoke/1-rook.yaml @@ -0,0 +1,7 @@ +tasks: +- rook: +- rook.shell: + - ceph -s + - ceph orch status + - ceph orch ps + - ceph orch ls diff --git a/ceph/qa/suites/orch/rook/smoke/2-workload/none.yaml b/ceph/qa/suites/orch/rook/smoke/2-workload/none.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/orch/rook/smoke/2-workload/radosbench.yaml b/ceph/qa/suites/orch/rook/smoke/2-workload/radosbench.yaml new file mode 100644 index 000000000..fd71605c8 --- /dev/null +++ b/ceph/qa/suites/orch/rook/smoke/2-workload/radosbench.yaml @@ -0,0 +1,5 @@ +tasks: +- install: + host.a: +- radosbench: + clients: [client.a] diff --git a/ceph/qa/suites/orch/rook/smoke/3-final.yaml b/ceph/qa/suites/orch/rook/smoke/3-final.yaml new file mode 100644 index 000000000..7628cc9da --- /dev/null +++ b/ceph/qa/suites/orch/rook/smoke/3-final.yaml @@ -0,0 +1,8 @@ +tasks: +- rook.shell: + commands: + - ceph orch status + - ceph orch ps + - ceph orch ls + - ceph orch host ls + - ceph orch device ls diff --git a/ceph/qa/suites/orch/rook/smoke/cluster/1-node.yaml b/ceph/qa/suites/orch/rook/smoke/cluster/1-node.yaml new file mode 100644 index 000000000..e238526ca --- /dev/null +++ b/ceph/qa/suites/orch/rook/smoke/cluster/1-node.yaml @@ -0,0 +1,3 @@ +roles: +- - host.a + - client.a diff --git a/ceph/qa/suites/orch/rook/smoke/cluster/3-node.yaml b/ceph/qa/suites/orch/rook/smoke/cluster/3-node.yaml new file mode 100644 index 000000000..d79a9f786 --- /dev/null +++ b/ceph/qa/suites/orch/rook/smoke/cluster/3-node.yaml @@ -0,0 +1,7 @@ +roles: +- - host.a + - client.a +- - host.b + - client.b +- - host.c + - client.c diff --git a/ceph/qa/suites/orch/rook/smoke/k8s/1.21.yaml b/ceph/qa/suites/orch/rook/smoke/k8s/1.21.yaml new file mode 100644 index 000000000..9e57a477f --- /dev/null +++ b/ceph/qa/suites/orch/rook/smoke/k8s/1.21.yaml @@ -0,0 +1,3 @@ +overrides: + kubeadm: + version: "1.21" diff --git a/ceph/qa/suites/orch/rook/smoke/net/calico.yaml b/ceph/qa/suites/orch/rook/smoke/net/calico.yaml new file mode 100644 index 000000000..7e838c6c8 --- /dev/null +++ b/ceph/qa/suites/orch/rook/smoke/net/calico.yaml @@ -0,0 +1,3 @@ +overrides: + kubeadm: + pod_network: calico diff --git a/ceph/qa/suites/orch/rook/smoke/rook/1.6.2.yaml b/ceph/qa/suites/orch/rook/smoke/rook/1.6.2.yaml new file mode 100644 index 000000000..2b9b25725 --- /dev/null +++ b/ceph/qa/suites/orch/rook/smoke/rook/1.6.2.yaml @@ -0,0 +1,4 @@ +overrides: + rook: + rook_image: rook/ceph:v1.6.2 + rook_branch: v1.6.2 diff --git a/ceph/qa/suites/orch/rook/smoke/rook/master.yaml b/ceph/qa/suites/orch/rook/smoke/rook/master.yaml new file mode 100644 index 000000000..72b1cec72 --- /dev/null +++ b/ceph/qa/suites/orch/rook/smoke/rook/master.yaml @@ -0,0 +1,3 @@ +overrides: + rook: + rook_image: rook/ceph:master diff --git a/ceph/qa/suites/rados/cephadm b/ceph/qa/suites/rados/cephadm new file mode 120000 index 000000000..0e8b6774d --- /dev/null +++ b/ceph/qa/suites/rados/cephadm @@ -0,0 +1 @@ +../orch/cephadm \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/thrash/3-tasks/rados_api_tests.yaml b/ceph/qa/suites/rados/cephadm/thrash/3-tasks/rados_api_tests.yaml deleted file mode 120000 index 47e335569..000000000 --- a/ceph/qa/suites/rados/cephadm/thrash/3-tasks/rados_api_tests.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../thrash/workloads/rados_api_tests.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/thrash/3-tasks/radosbench.yaml b/ceph/qa/suites/rados/cephadm/thrash/3-tasks/radosbench.yaml deleted file mode 120000 index 1e20917d8..000000000 --- a/ceph/qa/suites/rados/cephadm/thrash/3-tasks/radosbench.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../thrash/workloads/radosbench.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/thrash/3-tasks/small-objects.yaml b/ceph/qa/suites/rados/cephadm/thrash/3-tasks/small-objects.yaml deleted file mode 120000 index 8dbb9cd23..000000000 --- a/ceph/qa/suites/rados/cephadm/thrash/3-tasks/small-objects.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../thrash/workloads/small-objects.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/thrash/3-tasks/snaps-few-objects.yaml b/ceph/qa/suites/rados/cephadm/thrash/3-tasks/snaps-few-objects.yaml deleted file mode 120000 index 9ee934490..000000000 --- a/ceph/qa/suites/rados/cephadm/thrash/3-tasks/snaps-few-objects.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../thrash/workloads/snaps-few-objects.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/with-work/tasks/rados_api_tests.yaml b/ceph/qa/suites/rados/cephadm/with-work/tasks/rados_api_tests.yaml deleted file mode 120000 index c77107098..000000000 --- a/ceph/qa/suites/rados/cephadm/with-work/tasks/rados_api_tests.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../basic/tasks/rados_api_tests.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/with-work/tasks/rados_python.yaml b/ceph/qa/suites/rados/cephadm/with-work/tasks/rados_python.yaml deleted file mode 120000 index 31ac6eadb..000000000 --- a/ceph/qa/suites/rados/cephadm/with-work/tasks/rados_python.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../basic/tasks/rados_python.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rados/rook b/ceph/qa/suites/rados/rook new file mode 120000 index 000000000..1249cc92d --- /dev/null +++ b/ceph/qa/suites/rados/rook @@ -0,0 +1 @@ +../orch/rook \ No newline at end of file diff --git a/ceph/qa/suites/rados/standalone/mon_election b/ceph/qa/suites/rados/standalone/mon_election deleted file mode 120000 index 3f331e621..000000000 --- a/ceph/qa/suites/rados/standalone/mon_election +++ /dev/null @@ -1 +0,0 @@ -.qa/mon_election \ No newline at end of file diff --git a/ceph/qa/suites/rados/thrash/3-scrub-overrides/$ b/ceph/qa/suites/rados/thrash/3-scrub-overrides/$ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/rados/thrash/3-scrub-overrides/.qa b/ceph/qa/suites/rados/thrash/3-scrub-overrides/.qa new file mode 120000 index 000000000..fea2489fd --- /dev/null +++ b/ceph/qa/suites/rados/thrash/3-scrub-overrides/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/ceph/qa/suites/rados/thrash/3-scrub-overrides/default.yaml b/ceph/qa/suites/rados/thrash/3-scrub-overrides/default.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/rados/thrash/3-scrub-overrides/max-simultaneous-scrubs-2.yaml b/ceph/qa/suites/rados/thrash/3-scrub-overrides/max-simultaneous-scrubs-2.yaml new file mode 100644 index 000000000..abf852e98 --- /dev/null +++ b/ceph/qa/suites/rados/thrash/3-scrub-overrides/max-simultaneous-scrubs-2.yaml @@ -0,0 +1,5 @@ +overrides: + ceph: + conf: + osd: + osd max scrubs: 2 diff --git a/ceph/qa/suites/rados/thrash/3-scrub-overrides/max-simultaneous-scrubs-3.yaml b/ceph/qa/suites/rados/thrash/3-scrub-overrides/max-simultaneous-scrubs-3.yaml new file mode 100644 index 000000000..3b3dfd61f --- /dev/null +++ b/ceph/qa/suites/rados/thrash/3-scrub-overrides/max-simultaneous-scrubs-3.yaml @@ -0,0 +1,5 @@ +overrides: + ceph: + conf: + osd: + osd max scrubs: 3 diff --git a/ceph/qa/suites/rados/thrash/msgr-failures/osd-dispatch-delay.yaml b/ceph/qa/suites/rados/thrash/msgr-failures/osd-dispatch-delay.yaml new file mode 100644 index 000000000..aff059fb8 --- /dev/null +++ b/ceph/qa/suites/rados/thrash/msgr-failures/osd-dispatch-delay.yaml @@ -0,0 +1,7 @@ +overrides: + ceph: + conf: + global: + osd debug inject dispatch delay duration: 0.1 + osd debug inject dispatch delay probability: 0.1 + diff --git a/ceph/qa/suites/upgrade/octopus-x/parallel/1-tasks.yaml b/ceph/qa/suites/upgrade/octopus-x/parallel/1-tasks.yaml index c26119db5..3c298bc03 100644 --- a/ceph/qa/suites/upgrade/octopus-x/parallel/1-tasks.yaml +++ b/ceph/qa/suites/upgrade/octopus-x/parallel/1-tasks.yaml @@ -14,6 +14,7 @@ tasks: osd_class_default_list: "*" # deploy additional mons the "old" (octopus) way add_mons_via_daemon_add: true + avoid_pacific_features: true - print: "**** done end installing octopus cephadm ..." - cephadm.shell: diff --git a/ceph/qa/suites/upgrade/octopus-x/stress-split/1-start.yaml b/ceph/qa/suites/upgrade/octopus-x/stress-split/1-start.yaml index 1aa90967e..74599e22d 100644 --- a/ceph/qa/suites/upgrade/octopus-x/stress-split/1-start.yaml +++ b/ceph/qa/suites/upgrade/octopus-x/stress-split/1-start.yaml @@ -13,6 +13,7 @@ tasks: osd_class_default_list: "*" # deploy additional mons the "old" (octopus) way add_mons_via_daemon_add: true + avoid_pacific_features: true - cephadm.shell: mon.a: diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-parallel/% b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-parallel/% new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-parallel/point-to-point-upgrade.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-parallel/point-to-point-upgrade.yaml new file mode 100644 index 000000000..a433f607e --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-parallel/point-to-point-upgrade.yaml @@ -0,0 +1,177 @@ +meta: +- desc: | + Run ceph on two nodes, using one of them as a client, + with a separate client-only node. + Use xfs beneath the osds. + install ceph/pacific v16.2.2 and the v16.2.x point versions + run workload and upgrade-sequence in parallel + (every point release should be tested) + run workload and upgrade-sequence in parallel + install ceph/pacific latest version + run workload and upgrade-sequence in parallel + Overall upgrade path is - pacific-latest.point-1 => pacific-latest.point => pacific-latest +overrides: + ceph: + log-ignorelist: + - reached quota + - scrub + - osd_map_max_advance + - wrongly marked + - FS_DEGRADED + - POOL_APP_NOT_ENABLED + - CACHE_POOL_NO_HIT_SET + - POOL_FULL + - SMALLER_PG + - pool\(s\) full + - OSD_DOWN + - missing hit_sets + - CACHE_POOL_NEAR_FULL + - PG_AVAILABILITY + - PG_DEGRADED + - application not enabled + - cache pools at or near target size + - filesystem is degraded + - OBJECT_MISPLACED + ### ref: https://tracker.ceph.com/issues/40251 + #removed see ^ - failed to encode map + + fs: xfs + + conf: + global: + mon_warn_on_pool_no_app: false + mon: + mon debug unsafe allow tier with nonempty snaps: true + osd: + osd map max advance: 1000 + osd_class_default_list: "*" + osd_class_load_list: "*" + client: + rgw_crypt_require_ssl: false + rgw crypt s3 kms backend: testing + rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= +roles: +- - mon.a + - mds.a + - osd.0 + - osd.1 + - osd.2 + - mgr.x +- - mon.b + - mon.c + - osd.3 + - osd.4 + - osd.5 + - client.0 +- - client.1 +openstack: +- volumes: # attached to each instance + count: 3 + size: 30 # GB +tasks: +- print: "**** done pacific v16.2.0 about to install" +- install: + tag: v16.2.2 + # line below can be removed its from jewel test + #exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev', 'librgw2'] +- print: "**** done v16.2.2 install" +- ceph: + fs: xfs + add_osds_to_crush: true +- print: "**** done ceph xfs" +- sequential: + - workload +- print: "**** done workload v16.2.2" + + +####### upgrade to v16.2.3 +- install.upgrade: + #exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev'] + mon.a: + tag: v16.2.3 + mon.b: + tag: v16.2.3 +- parallel: + - workload_pacific + - upgrade-sequence_pacific +- print: "**** done parallel pacific v16.2.3" + +#### upgrade to latest pacific +- install.upgrade: + mon.a: + mon.b: +- parallel: + - workload_pacific + - upgrade-sequence_pacific +- print: "**** done parallel pacific branch" + +####################### +workload: + sequential: + - workunit: + clients: + client.0: + - suites/blogbench.sh +workload_pacific: + full_sequential: + - workunit: + branch: pacific + #tag: v16.2.1 + clients: + client.1: + - rados/test.sh + - cls + env: + CLS_RBD_GTEST_FILTER: '*:-TestClsRbd.snapshots_namespaces' + - print: "**** done rados/test.sh & cls workload_pacific" + - sequential: + - rgw: [client.0] + - print: "**** done rgw workload_pacific" + - s3tests: + client.0: + force-branch: ceph-pacific + rgw_server: client.0 + scan_for_encryption_keys: false + - print: "**** done s3tests workload_pacific" + - rbd_fsx: + clients: [client.0] + size: 134217728 + - print: "**** done rbd_fsx workload_pacific" + +upgrade-sequence_pacific: + sequential: + - print: "**** done branch: pacific install.upgrade" + - ceph.restart: [mds.a] + - sleep: + duration: 60 + - ceph.restart: [osd.0] + - sleep: + duration: 30 + - ceph.restart: [osd.1] + - sleep: + duration: 30 + - ceph.restart: [osd.2] + - sleep: + duration: 30 + - ceph.restart: [osd.3] + - sleep: + duration: 30 + - ceph.restart: [osd.4] + - sleep: + duration: 30 + - ceph.restart: [osd.5] + - sleep: + duration: 60 + - ceph.restart: [mgr.x] + - sleep: + duration: 60 + - ceph.restart: [mon.a] + - sleep: + duration: 60 + - ceph.restart: [mon.b] + - sleep: + duration: 60 + - ceph.restart: [mon.c] + - sleep: + duration: 60 + - print: "**** done ceph.restart all pacific branch mds/osd/mon" diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-parallel/supported-all-distro/ubuntu_latest.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-parallel/supported-all-distro/ubuntu_latest.yaml new file mode 100644 index 000000000..f20398230 --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-parallel/supported-all-distro/ubuntu_latest.yaml @@ -0,0 +1,2 @@ +os_type: ubuntu +os_version: "20.04" diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/% b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/% new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/0-cluster/+ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/0-cluster/+ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/0-cluster/openstack.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/0-cluster/openstack.yaml new file mode 100644 index 000000000..5caffc353 --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/0-cluster/openstack.yaml @@ -0,0 +1,6 @@ +openstack: + - machine: + disk: 100 # GB + - volumes: # attached to each instance + count: 4 + size: 30 # GB diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/0-cluster/start.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/0-cluster/start.yaml new file mode 100644 index 000000000..1271edd8b --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/0-cluster/start.yaml @@ -0,0 +1,33 @@ +meta: +- desc: | + Run ceph on two nodes, + with a separate client-only node. + Use xfs beneath the osds. +overrides: + ceph: + fs: xfs + log-ignorelist: + - overall HEALTH_ + - \(MON_DOWN\) + - \(MGR_DOWN\) + ### ref: https://tracker.ceph.com/issues/40251 + #removed see ^ - failed to encode map + conf: + global: + enable experimental unrecoverable data corrupting features: "*" + mon: + mon warn on osd down out interval zero: false +roles: +- - mon.a + - mon.b + - mon.c + - mgr.x + - osd.0 + - osd.1 + - osd.2 + - osd.3 +- - osd.4 + - osd.5 + - osd.6 + - osd.7 +- - client.0 diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/1-ceph-install/pacific..yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/1-ceph-install/pacific..yaml new file mode 100644 index 000000000..2de7badc4 --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/1-ceph-install/pacific..yaml @@ -0,0 +1,21 @@ +meta: +- desc: | + install ceph/pacific v16.2.3 + Overall upgrade path is - pacific-latest.point => pacific-latest +tasks: +- install: + tag: v16.2.3 + exclude_packages: ['librados3'] + extra_packages: ['librados2'] +- print: "**** done install pacific v16.2.3" +- ceph: +- exec: + osd.0: + - ceph osd require-osd-release pacific + - ceph osd set-require-min-compat-client pacific +- print: "**** done ceph" +overrides: + ceph: + conf: + mon: + mon warn on osd down out interval zero: false diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/1.1.short_pg_log.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/1.1.short_pg_log.yaml new file mode 100644 index 000000000..20cc101de --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/1.1.short_pg_log.yaml @@ -0,0 +1,6 @@ +overrides: + ceph: + conf: + global: + osd_min_pg_log_entries: 1 + osd_max_pg_log_entries: 2 diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/2-partial-upgrade/firsthalf.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/2-partial-upgrade/firsthalf.yaml new file mode 100644 index 000000000..02ba5c1bb --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/2-partial-upgrade/firsthalf.yaml @@ -0,0 +1,13 @@ +meta: +- desc: | + install upgrade ceph/-x on one node only + 1st half + restart : osd.0,1,2,3 +tasks: +- install.upgrade: + osd.0: +- print: "**** done install.upgrade osd.0" +- ceph.restart: + daemons: [mon.a,mon.b,mon.c,mgr.x,osd.0,osd.1,osd.2,osd.3] + mon-health-to-clog: false +- print: "**** done ceph.restart 1st half" diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/3-thrash/default.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/3-thrash/default.yaml new file mode 100644 index 000000000..c739d8fea --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/3-thrash/default.yaml @@ -0,0 +1,27 @@ +meta: +- desc: | + randomly kill and revive osd + small chance to increase the number of pgs +overrides: + ceph: + log-ignorelist: + - but it is still running + - wrongly marked me down + - objects unfound and apparently lost + - log bound mismatch + ### ref: https://tracker.ceph.com/issues/40251 + - failed to encode map +tasks: +- parallel: + - stress-tasks +stress-tasks: +- thrashosds: + timeout: 1200 + chance_pgnum_grow: 1 + chance_pgpnum_fix: 1 + chance_thrash_cluster_full: 0 + chance_thrash_pg_upmap: 0 + chance_thrash_pg_upmap_items: 0 + disable_objectstore_tool_tests: true + chance_force_recovery: 0 +- print: "**** done thrashosds 3-thrash" diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/+ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/+ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/fsx.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/fsx.yaml new file mode 100644 index 000000000..fd4081f23 --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/fsx.yaml @@ -0,0 +1,8 @@ +meta: +- desc: | + run basic fsx tests for rbd +stress-tasks: +- rbd_fsx: + clients: [client.0] + size: 134217728 +- print: "**** done rbd_fsx 4-workload" diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/radosbench.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/radosbench.yaml new file mode 100644 index 000000000..c545936c0 --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/radosbench.yaml @@ -0,0 +1,52 @@ +meta: +- desc: | + run randomized correctness test for rados operations + generate write load with rados bench +stress-tasks: +- full_sequential: + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 +- print: "**** done radosbench 4-workload" diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/rbd-cls.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/rbd-cls.yaml new file mode 100644 index 000000000..caaac875c --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/rbd-cls.yaml @@ -0,0 +1,10 @@ +meta: +- desc: | + run basic cls tests for rbd +stress-tasks: +- workunit: + branch: pacific + clients: + client.0: + - cls/test_cls_rbd.sh +- print: "**** done cls/test_cls_rbd.sh 4-workload" diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/rbd-import-export.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/rbd-import-export.yaml new file mode 100644 index 000000000..f999bd0c8 --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/rbd-import-export.yaml @@ -0,0 +1,12 @@ +meta: +- desc: | + run basic import/export cli tests for rbd +stress-tasks: +- workunit: + branch: pacific + clients: + client.0: + - rbd/import_export.sh + env: + RBD_CREATE_ARGS: --new-format +- print: "**** done rbd/import_export.sh 4-workload" diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/rbd_api.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/rbd_api.yaml new file mode 100644 index 000000000..95c820161 --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/rbd_api.yaml @@ -0,0 +1,10 @@ +meta: +- desc: | + librbd C and C++ api tests +stress-tasks: +- workunit: + branch: octopus + clients: + client.0: + - rbd/test_librbd.sh +- print: "**** done rbd/test_librbd.sh 4-workload" diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/readwrite.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/readwrite.yaml new file mode 100644 index 000000000..456868998 --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/readwrite.yaml @@ -0,0 +1,16 @@ +meta: +- desc: | + randomized correctness test for rados operations on a replicated pool, + using only reads, writes, and deletes +stress-tasks: +- full_sequential: + - rados: + clients: [client.0] + ops: 4000 + objects: 500 + write_append_excl: false + op_weights: + read: 45 + write: 45 + delete: 10 +- print: "**** done rados/readwrite 4-workload" diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/snaps-few-objects.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/snaps-few-objects.yaml new file mode 100644 index 000000000..ae232d867 --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/4-workload/snaps-few-objects.yaml @@ -0,0 +1,18 @@ +meta: +- desc: | + randomized correctness test for rados operations on a replicated pool with snapshot operations +stress-tasks: +- full_sequential: + - rados: + clients: [client.0] + ops: 4000 + objects: 50 + write_append_excl: false + op_weights: + read: 100 + write: 100 + delete: 50 + snap_create: 50 + snap_remove: 50 + rollback: 50 +- print: "**** done rados/snaps-few-objects 4-workload" diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/5-finish-upgrade.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/5-finish-upgrade.yaml new file mode 100644 index 000000000..803737c72 --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/5-finish-upgrade.yaml @@ -0,0 +1,8 @@ +tasks: +- install.upgrade: + osd.4: + client.0: +- ceph.restart: + daemons: [osd.4, osd.5, osd.6, osd.7] + wait-for-healthy: false + wait-for-osds-up: true diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/6-final-workload/+ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/6-final-workload/+ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/6-final-workload/rbd-python.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/6-final-workload/rbd-python.yaml new file mode 100644 index 000000000..4578b4dac --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/6-final-workload/rbd-python.yaml @@ -0,0 +1,10 @@ +meta: +- desc: | + librbd python api tests +tasks: +- workunit: + tag: v16.2.0 + clients: + client.0: + - rbd/test_librbd_python.sh +- print: "**** done rbd/test_librbd_python.sh 7-workload" diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/6-final-workload/snaps-many-objects.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/6-final-workload/snaps-many-objects.yaml new file mode 100644 index 000000000..805bf97c3 --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/6-final-workload/snaps-many-objects.yaml @@ -0,0 +1,16 @@ +meta: +- desc: | + randomized correctness test for rados operations on a replicated pool with snapshot operations +tasks: +- rados: + clients: [client.0] + ops: 4000 + objects: 500 + write_append_excl: false + op_weights: + read: 100 + write: 100 + delete: 50 + snap_create: 50 + snap_remove: 50 + rollback: 50 diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/objectstore/bluestore-bitmap.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/objectstore/bluestore-bitmap.yaml new file mode 100644 index 000000000..b18e04bee --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/objectstore/bluestore-bitmap.yaml @@ -0,0 +1,43 @@ +overrides: + thrashosds: + bdev_inject_crash: 2 + bdev_inject_crash_probability: .5 + ceph: + fs: xfs + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore fsck on mount: true + bluestore allocator: bitmap + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 +# this doesn't work with failures bc the log writes are not atomic across the two backends +# bluestore bluefs env mirror: true + bdev enable discard: true + bdev async discard: true + ceph-deploy: + fs: xfs + bluestore: yes + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore fsck on mount: true + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 + bdev enable discard: true + bdev async discard: true + diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/objectstore/bluestore-comp.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/objectstore/bluestore-comp.yaml new file mode 100644 index 000000000..b408032fd --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/objectstore/bluestore-comp.yaml @@ -0,0 +1,23 @@ +overrides: + thrashosds: + bdev_inject_crash: 2 + bdev_inject_crash_probability: .5 + ceph: + fs: xfs + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore compression mode: aggressive + bluestore fsck on mount: true + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 + +# this doesn't work with failures bc the log writes are not atomic across the two backends +# bluestore bluefs env mirror: true diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/objectstore/bluestore-stupid.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/objectstore/bluestore-stupid.yaml new file mode 100644 index 000000000..ca811f131 --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/objectstore/bluestore-stupid.yaml @@ -0,0 +1,43 @@ +overrides: + thrashosds: + bdev_inject_crash: 2 + bdev_inject_crash_probability: .5 + ceph: + fs: xfs + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore fsck on mount: true + bluestore allocator: stupid + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 +# this doesn't work with failures bc the log writes are not atomic across the two backends +# bluestore bluefs env mirror: true + bdev enable discard: true + bdev async discard: true + ceph-deploy: + fs: xfs + bluestore: yes + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore fsck on mount: true + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 + bdev enable discard: true + bdev async discard: true + diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/objectstore/filestore-xfs.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/objectstore/filestore-xfs.yaml new file mode 100644 index 000000000..f7aa0dd79 --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/objectstore/filestore-xfs.yaml @@ -0,0 +1,15 @@ +overrides: + ceph: + fs: xfs + conf: + osd: + osd objectstore: filestore + osd sloppy crc: true + ceph-deploy: + fs: xfs + filestore: True + conf: + osd: + osd objectstore: filestore + osd sloppy crc: true + diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/supported-all-distro/ubuntu_latest.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/supported-all-distro/ubuntu_latest.yaml new file mode 100644 index 000000000..f20398230 --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/supported-all-distro/ubuntu_latest.yaml @@ -0,0 +1,2 @@ +os_type: ubuntu +os_version: "20.04" diff --git a/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/thrashosds-health.yaml b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/thrashosds-health.yaml new file mode 100644 index 000000000..9903fa578 --- /dev/null +++ b/ceph/qa/suites/upgrade/pacific-p2p/pacific-p2p-stress-split/thrashosds-health.yaml @@ -0,0 +1,15 @@ +overrides: + ceph: + log-ignorelist: + - overall HEALTH_ + - \(OSDMAP_FLAGS\) + - \(OSD_ + - \(PG_ + - \(POOL_ + - \(CACHE_POOL_ + - \(SMALLER_PGP_NUM\) + - \(OBJECT_ + - \(SLOW_OPS\) + - \(REQUEST_SLOW\) + - \(TOO_FEW_PGS\) + - slow request diff --git a/ceph/qa/tasks/ceph_fuse.py b/ceph/qa/tasks/ceph_fuse.py index 405742891..d2db29732 100644 --- a/ceph/qa/tasks/ceph_fuse.py +++ b/ceph/qa/tasks/ceph_fuse.py @@ -108,9 +108,7 @@ def task(ctx, config): if client_config is None: client_config = {} # top level overrides - for k, v in top_overrides.items(): - if v is not None: - client_config[k] = v + misc.deep_merge(client_config, top_overrides) # mount specific overrides client_config_overrides = overrides.get(entity) misc.deep_merge(client_config, client_config_overrides) diff --git a/ceph/qa/tasks/ceph_manager.py b/ceph/qa/tasks/ceph_manager.py index e67ca06dd..1d1f9049e 100644 --- a/ceph/qa/tasks/ceph_manager.py +++ b/ceph/qa/tasks/ceph_manager.py @@ -50,6 +50,20 @@ def shell(ctx, cluster_name, remote, args, name=None, **kwargs): **kwargs ) +# this is for rook clusters +def toolbox(ctx, cluster_name, args, **kwargs): + return ctx.rook[cluster_name].remote.run( + args=[ + 'kubectl', + '-n', 'rook-ceph', + 'exec', + ctx.rook[cluster_name].toolbox, + '--', + ] + args, + **kwargs + ) + + def write_conf(ctx, conf_path=DEFAULT_CONF_PATH, cluster='ceph'): conf_fp = BytesIO() ctx.ceph[cluster].conf.write(conf_fp) @@ -267,7 +281,7 @@ class OSDThrasher(Thrasher): self.logger.info(msg, *args, **kwargs) def cmd_exists_on_osds(self, cmd): - if self.ceph_manager.cephadm: + if self.ceph_manager.cephadm or self.ceph_manager.rook: return True allremotes = self.ceph_manager.ctx.cluster.only(\ teuthology.is_type('osd', self.cluster)).remotes.keys() @@ -289,6 +303,8 @@ class OSDThrasher(Thrasher): wait=True, check_status=False, stdout=StringIO(), stderr=StringIO()) + elif self.ceph_manager.rook: + assert False, 'not implemented' else: return remote.run( args=['sudo', 'adjust-ulimits', 'ceph-objectstore-tool'] + cmd, @@ -305,6 +321,8 @@ class OSDThrasher(Thrasher): wait=True, check_status=False, stdout=StringIO(), stderr=StringIO()) + elif self.ceph_manager.rook: + assert False, 'not implemented' else: return remote.run( args=['sudo', 'ceph-bluestore-tool', '--err-to-stderr'] + cmd, @@ -348,6 +366,9 @@ class OSDThrasher(Thrasher): '--log-file=/var/log/ceph/objectstore_tool.$pid.log', ] + if self.ceph_manager.rook: + assert False, 'not implemented' + if not self.ceph_manager.cephadm: # ceph-objectstore-tool might be temporarily absent during an # upgrade - see http://tracker.ceph.com/issues/18014 @@ -1486,7 +1507,7 @@ class CephManager: """ def __init__(self, controller, ctx=None, config=None, logger=None, - cluster='ceph', cephadm=False) -> None: + cluster='ceph', cephadm=False, rook=False) -> None: self.lock = threading.RLock() self.ctx = ctx self.config = config @@ -1494,6 +1515,7 @@ class CephManager: self.next_pool_id = 0 self.cluster = cluster self.cephadm = cephadm + self.rook = rook if (logger): self.log = lambda x: logger.info(x) else: @@ -1537,6 +1559,11 @@ class CephManager: args=['ceph'] + list(kwargs['args']), stdout=StringIO(), check_status=kwargs.get('check_status', True)) + if self.rook: + return toolbox(self.ctx, self.cluster, + args=['ceph'] + list(kwargs['args']), + stdout=StringIO(), + check_status=kwargs.get('check_status', True)) testdir = teuthology.get_testdir(self.ctx) prefix = ['sudo', 'adjust-ulimits', 'ceph-coverage', @@ -1789,6 +1816,8 @@ class CephManager: wait=True, check_status=check_status, ) + if self.rook: + assert False, 'not implemented' testdir = teuthology.get_testdir(self.ctx) args = [ diff --git a/ceph/qa/tasks/cephadm.conf b/ceph/qa/tasks/cephadm.conf index 4a7945ac3..bd1ab821e 100644 --- a/ceph/qa/tasks/cephadm.conf +++ b/ceph/qa/tasks/cephadm.conf @@ -46,6 +46,8 @@ osd deep scrub update digest min age = 30 osd map max advance = 10 +osd memory target autotune = true + # debugging osd debug shutdown = true osd debug op order = true diff --git a/ceph/qa/tasks/cephadm.py b/ceph/qa/tasks/cephadm.py index c1b6970c8..57ede1813 100644 --- a/ceph/qa/tasks/cephadm.py +++ b/ceph/qa/tasks/cephadm.py @@ -46,6 +46,7 @@ def _shell(ctx, cluster_name, remote, args, extra_cephadm_args=[], **kwargs): **kwargs ) + def build_initial_config(ctx, config): cluster_name = config['cluster'] @@ -65,10 +66,13 @@ def build_initial_config(ctx, config): return conf + def update_archive_setting(ctx, key, value): """ Add logs directory to job's info log file """ + if ctx.archive is None: + return with open(os.path.join(ctx.archive, 'info.yaml'), 'r+') as info_file: info_yaml = yaml.safe_load(info_file) info_file.seek(0) @@ -78,6 +82,7 @@ def update_archive_setting(ctx, key, value): info_yaml['archive'] = {key: value} yaml.safe_dump(info_yaml, info_file, default_flow_style=False) + @contextlib.contextmanager def normalize_hostnames(ctx): """ @@ -96,6 +101,7 @@ def normalize_hostnames(ctx): finally: pass + @contextlib.contextmanager def download_cephadm(ctx, config, ref): cluster_name = config['cluster'] @@ -104,7 +110,20 @@ def download_cephadm(ctx, config, ref): ref = config.get('cephadm_branch', ref) git_url = config.get('cephadm_git_url', teuth_config.get_ceph_git_url()) log.info('Downloading cephadm (repo %s ref %s)...' % (git_url, ref)) - if git_url.startswith('https://github.com/'): + if ctx.config.get('redhat'): + log.info("Install cephadm using RPM") + # cephadm already installed from redhat.install task + ctx.cluster.run( + args=[ + 'cp', + run.Raw('$(which cephadm)'), + ctx.cephadm, + run.Raw('&&'), + 'ls', '-l', + ctx.cephadm, + ] + ) + elif git_url.startswith('https://github.com/'): # git archive doesn't like https:// URLs, which we use with github. rest = git_url.split('https://github.com/', 1)[1] rest = re.sub(r'\.git/?$', '', rest).strip() # no .git suffix @@ -166,6 +185,7 @@ def download_cephadm(ctx, config, ref): ], ) + @contextlib.contextmanager def ceph_log(ctx, config): cluster_name = config['cluster'] @@ -274,6 +294,7 @@ def ceph_log(ctx, config): except ReadError: pass + @contextlib.contextmanager def ceph_crash(ctx, config): """ @@ -308,6 +329,7 @@ def ceph_crash(ctx, config): except ReadError: pass + @contextlib.contextmanager def ceph_bootstrap(ctx, config): """ @@ -378,6 +400,15 @@ def ceph_bootstrap(ctx, config): '/etc/ceph/{}.client.admin.keyring'.format(cluster_name), '--output-pub-ssh-key', '{}/{}.pub'.format(testdir, cluster_name), ] + + if config.get('registry-login'): + registry = config['registry-login'] + cmd += [ + "--registry-url", registry['url'], + "--registry-username", registry['username'], + "--registry-password", registry['password'], + ] + if not ctx.ceph[cluster_name].roleless: cmd += [ '--mon-id', first_mon, @@ -385,6 +416,7 @@ def ceph_bootstrap(ctx, config): '--orphan-initial-daemons', # we will do it explicitly! '--skip-monitoring-stack', # we'll provision these explicitly ] + if mons[first_mon_role].startswith('['): cmd += ['--mon-addrv', mons[first_mon_role]] else: @@ -393,6 +425,10 @@ def ceph_bootstrap(ctx, config): cmd += ['--skip-dashboard'] if config.get('skip_monitoring_stack'): cmd += ['--skip-monitoring-stack'] + if config.get('single_host_defaults'): + cmd += ['--single-host-defaults'] + if not config.get('avoid_pacific_features', False): + cmd += ['--skip-admin-label'] # bootstrap makes the keyring root 0600, so +r it for our purposes cmd += [ run.Raw('&&'), @@ -433,10 +469,20 @@ def ceph_bootstrap(ctx, config): _shell(ctx, cluster_name, bootstrap_remote, ['ceph', 'config', 'set', 'mgr', 'mgr/cephadm/allow_ptrace', 'true']) + if not config.get('avoid_pacific_features', False): + log.info('Distributing conf and client.admin keyring to all hosts + 0755') + _shell(ctx, cluster_name, bootstrap_remote, + ['ceph', 'orch', 'client-keyring', 'set', 'client.admin', + '*', '--mode', '0755'], + check_status=False) + # add other hosts for remote in ctx.cluster.remotes.keys(): if remote == bootstrap_remote: continue + + # note: this may be redundant (see above), but it avoids + # us having to wait for cephadm to do it. log.info('Writing (initial) conf and keyring to %s' % remote.shortname) remote.write_file( path='/etc/ceph/{}.conf'.format(cluster_name), @@ -500,6 +546,7 @@ def ceph_bootstrap(ctx, config): '/etc/ceph/{}.client.admin.keyring'.format(cluster_name), ]) + @contextlib.contextmanager def ceph_mons(ctx, config): """ @@ -615,6 +662,7 @@ def ceph_mons(ctx, config): finally: pass + @contextlib.contextmanager def ceph_mgrs(ctx, config): """ @@ -656,6 +704,7 @@ def ceph_mgrs(ctx, config): finally: pass + @contextlib.contextmanager def ceph_osds(ctx, config): """ @@ -711,6 +760,7 @@ def ceph_osds(ctx, config): finally: pass + @contextlib.contextmanager def ceph_mdss(ctx, config): """ @@ -747,6 +797,7 @@ def ceph_mdss(ctx, config): yield + @contextlib.contextmanager def ceph_monitoring(daemon_type, ctx, config): """ @@ -782,6 +833,7 @@ def ceph_monitoring(daemon_type, ctx, config): yield + @contextlib.contextmanager def ceph_rgw(ctx, config): """ @@ -872,6 +924,7 @@ def ceph_iscsi(ctx, config): yield + @contextlib.contextmanager def ceph_clients(ctx, config): cluster_name = config['cluster'] @@ -902,6 +955,7 @@ def ceph_clients(ctx, config): remote.sudo_write_file(client_keyring, keyring, mode='0644') yield + @contextlib.contextmanager def ceph_initial(): try: @@ -909,6 +963,7 @@ def ceph_initial(): finally: log.info('Teardown complete') + ## public methods @contextlib.contextmanager def stop(ctx, config): @@ -947,17 +1002,18 @@ def stop(ctx, config): yield + def shell(ctx, config): """ Execute (shell) commands """ cluster_name = config.get('cluster', 'ceph') - env = [] - if 'env' in config: - for k in config['env']: - env.extend(['-e', k + '=' + ctx.config.get(k, '')]) - del config['env'] + args = [] + for k in config.pop('env', []): + args.extend(['-e', k + '=' + ctx.config.get(k, '')]) + for k in config.pop('volumes', []): + args.extend(['-v', k]) if 'all-roles' in config and len(config) == 1: a = config['all-roles'] @@ -975,12 +1031,12 @@ def shell(ctx, config): for c in cmd: _shell(ctx, cluster_name, remote, ['bash', '-c', subst_vip(ctx, c)], - extra_cephadm_args=env) + extra_cephadm_args=args) else: assert isinstance(cmd, str) _shell(ctx, cluster_name, remote, ['bash', '-ex', '-c', subst_vip(ctx, cmd)], - extra_cephadm_args=env) + extra_cephadm_args=args) def apply(ctx, config): @@ -1090,6 +1146,7 @@ def tweaked_option(ctx, config): for option, value in saved_options.items(): manager.inject_args(type_, id_, option, value) + @contextlib.contextmanager def restart(ctx, config): """ @@ -1143,6 +1200,7 @@ def restart(ctx, config): ctx.managers[cluster].wait_for_all_osds_up() yield + @contextlib.contextmanager def distribute_config_and_admin_keyring(ctx, config): """ @@ -1168,6 +1226,7 @@ def distribute_config_and_admin_keyring(ctx, config): '/etc/ceph/{}.client.admin.keyring'.format(cluster_name), ]) + @contextlib.contextmanager def crush_setup(ctx, config): cluster_name = config['cluster'] @@ -1178,6 +1237,7 @@ def crush_setup(ctx, config): args=['ceph', 'osd', 'crush', 'tunables', profile]) yield + @contextlib.contextmanager def create_rbd_pool(ctx, config): if config.get('create_rbd_pool', False): @@ -1200,10 +1260,12 @@ def create_rbd_pool(ctx, config): ]) yield + @contextlib.contextmanager def _bypass(): yield + @contextlib.contextmanager def initialize_config(ctx, config): cluster_name = config['cluster'] @@ -1287,6 +1349,7 @@ def initialize_config(ctx, config): ctx.ceph[cluster_name].first_mgr = first_mgr yield + @contextlib.contextmanager def task(ctx, config): """ @@ -1306,6 +1369,10 @@ def task(ctx, config): cephadm: containers: image: 'quay.io/ceph-ci/ceph' + registry-login: + url: registry-url + username: registry-user + password: registry-password :param ctx: the argparse.Namespace object :param config: the config dict @@ -1324,7 +1391,6 @@ def task(ctx, config): # set up cluster context if not hasattr(ctx, 'ceph'): ctx.ceph = {} - ctx.managers = {} if 'cluster' not in config: config['cluster'] = 'ceph' cluster_name = config['cluster'] @@ -1341,7 +1407,6 @@ def task(ctx, config): containers = config.get('containers', {}) container_image_name = containers.get('image', container_image_name) - if not hasattr(ctx.ceph[cluster_name], 'image'): ctx.ceph[cluster_name].image = config.get('image') ref = None @@ -1396,6 +1461,8 @@ def task(ctx, config): lambda: ceph_clients(ctx=ctx, config=config), lambda: create_rbd_pool(ctx=ctx, config=config), ): + if not hasattr(ctx, 'managers'): + ctx.managers = {} ctx.managers[cluster_name] = CephManager( ctx.ceph[cluster_name].bootstrap_remote, ctx=ctx, diff --git a/ceph/qa/tasks/cephadm_cases/test_cli.py b/ceph/qa/tasks/cephadm_cases/test_cli.py index f96359374..1dcf2f35e 100644 --- a/ceph/qa/tasks/cephadm_cases/test_cli.py +++ b/ceph/qa/tasks/cephadm_cases/test_cli.py @@ -49,7 +49,7 @@ class TestCephadmCLI(MgrTestCase): self._orch_cmd('daemon', 'stop', 'osd.0') self.wait_for_health('OSD_DOWN', 30) self._orch_cmd('daemon', 'start', 'osd.0') - self.wait_for_health_clear(30) + self.wait_for_health_clear(90) self._orch_cmd('daemon', 'restart', 'osd.0') def test_device_ls_wide(self): diff --git a/ceph/qa/tasks/cephfs/test_data_scan.py b/ceph/qa/tasks/cephfs/test_data_scan.py index 7d50f1003..933a7f67d 100644 --- a/ceph/qa/tasks/cephfs/test_data_scan.py +++ b/ceph/qa/tasks/cephfs/test_data_scan.py @@ -504,7 +504,7 @@ class TestDataScan(CephFSTestCase): # run scrub to update and make sure rstat.rbytes info in subdir inode and dirfrag # are matched - out_json = self.fs.run_scrub(["start", "/subdir", "repair", "recursive"]) + out_json = self.fs.run_scrub(["start", "/subdir", "repair,recursive"]) self.assertNotEqual(out_json, None) self.assertEqual(out_json["return_code"], 0) self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) @@ -545,7 +545,7 @@ class TestDataScan(CephFSTestCase): pg_count = self.fs.pgs_per_fs_pool for pg_n in range(0, pg_count): - pg_str = "{0}.{1}".format(self.fs.get_data_pool_id(), pg_n) + pg_str = "{0}.{1:x}".format(self.fs.get_data_pool_id(), pg_n) out = self.fs.data_scan(["pg_files", "mydir", pg_str]) lines = [l for l in out.split("\n") if l] log.info("{0}: {1}".format(pg_str, lines)) diff --git a/ceph/qa/tasks/cephfs/test_exports.py b/ceph/qa/tasks/cephfs/test_exports.py index 36d26befe..d2421bedc 100644 --- a/ceph/qa/tasks/cephfs/test_exports.py +++ b/ceph/qa/tasks/cephfs/test_exports.py @@ -181,7 +181,7 @@ rm -rf .inode_number_thrash def _setup_tree(self, path="tree", export=-1, distributed=False, random=0.0, count=100, wait=True): return self.mount_a.run_shell_payload(f""" -set -e +set -ex mkdir -p {path} {f"setfattr -n ceph.dir.pin -v {export} {path}" if export >= 0 else ""} {f"setfattr -n ceph.dir.pin.distributed -v 1 {path}" if distributed else ""} @@ -316,22 +316,25 @@ done That ephemerally pinned subtrees are somewhat evenly distributed. """ - self.fs.set_max_mds(3) + max_mds = 3 + frags = 128 + + self.fs.set_max_mds(max_mds) self.status = self.fs.wait_for_daemons() - self.config_set('mds', 'mds_export_ephemeral_distributed_factor', 63.0 / 3) + self.config_set('mds', 'mds_export_ephemeral_distributed_factor', (frags-1) / max_mds) self._setup_tree(count=1000, distributed=True) - subtrees = self._wait_distributed_subtrees(64, status=self.status, rank="all") + subtrees = self._wait_distributed_subtrees(frags, status=self.status, rank="all") nsubtrees = len(subtrees) # Check if distribution is uniform rank0 = list(filter(lambda x: x['auth_first'] == 0, subtrees)) rank1 = list(filter(lambda x: x['auth_first'] == 1, subtrees)) rank2 = list(filter(lambda x: x['auth_first'] == 2, subtrees)) - self.assertGreaterEqual(len(rank0)/nsubtrees, 0.2) - self.assertGreaterEqual(len(rank1)/nsubtrees, 0.2) - self.assertGreaterEqual(len(rank2)/nsubtrees, 0.2) + self.assertGreaterEqual(len(rank0)/nsubtrees, 0.15) + self.assertGreaterEqual(len(rank1)/nsubtrees, 0.15) + self.assertGreaterEqual(len(rank2)/nsubtrees, 0.15) def test_ephemeral_random(self): diff --git a/ceph/qa/tasks/cephfs/test_forward_scrub.py b/ceph/qa/tasks/cephfs/test_forward_scrub.py index 19eb4afaf..82630e069 100644 --- a/ceph/qa/tasks/cephfs/test_forward_scrub.py +++ b/ceph/qa/tasks/cephfs/test_forward_scrub.py @@ -232,7 +232,7 @@ class TestForwardScrub(CephFSTestCase): self.mount_a.umount_wait() with self.assert_cluster_log("inode table repaired", invert_match=True): - out_json = self.fs.run_scrub(["start", "/", "repair", "recursive"]) + out_json = self.fs.run_scrub(["start", "/", "repair,recursive"]) self.assertNotEqual(out_json, None) self.assertEqual(out_json["return_code"], 0) self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) @@ -256,7 +256,7 @@ class TestForwardScrub(CephFSTestCase): self.fs.wait_for_daemons() with self.assert_cluster_log("inode table repaired"): - out_json = self.fs.run_scrub(["start", "/", "repair", "recursive"]) + out_json = self.fs.run_scrub(["start", "/", "repair,recursive"]) self.assertNotEqual(out_json, None) self.assertEqual(out_json["return_code"], 0) self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) @@ -289,7 +289,7 @@ class TestForwardScrub(CephFSTestCase): "oh i'm sorry did i overwrite your xattr?") with self.assert_cluster_log("bad backtrace on inode"): - out_json = self.fs.run_scrub(["start", "/", "repair", "recursive"]) + out_json = self.fs.run_scrub(["start", "/", "repair,recursive"]) self.assertNotEqual(out_json, None) self.assertEqual(out_json["return_code"], 0) self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) diff --git a/ceph/qa/tasks/cephfs/test_mirroring.py b/ceph/qa/tasks/cephfs/test_mirroring.py index e2f9b72ab..6ea3d5d30 100644 --- a/ceph/qa/tasks/cephfs/test_mirroring.py +++ b/ceph/qa/tasks/cephfs/test_mirroring.py @@ -2,9 +2,11 @@ import os import json import errno import logging +import random import time from io import StringIO +from collections import deque from tasks.cephfs.cephfs_test_case import CephFSTestCase from teuthology.exceptions import CommandFailedError @@ -24,6 +26,7 @@ class TestMirroring(CephFSTestCase): self.primary_fs_name = self.fs.name self.primary_fs_id = self.fs.id self.secondary_fs_name = self.backup_fs.name + self.secondary_fs_id = self.backup_fs.id self.enable_mirroring_module() def tearDown(self): @@ -234,25 +237,11 @@ class TestMirroring(CephFSTestCase): log.debug(f'command returned={res}') return json.loads(res) - def get_mirror_daemon_id(self): - ceph_status = json.loads(self.fs.mon_manager.raw_cluster_cmd("status", "--format=json")) - log.debug(f'ceph_status: {ceph_status}') - daemon_id = None - for k in ceph_status['servicemap']['services']['cephfs-mirror']['daemons']: - try: - daemon_id = int(k) - break #nit, only a single mirror daemon is expected -- bail out. - except ValueError: - pass - - log.debug(f'daemon_id: {daemon_id}') - self.assertTrue(daemon_id is not None) - return daemon_id - - def get_mirror_daemon_status(self, daemon_id, fs_name, fs_id): + def get_mirror_daemon_status(self, fs_name, fs_id): daemon_status = json.loads(self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", "snapshot", "mirror", "daemon", "status", fs_name)) log.debug(f'daemon_status: {daemon_status}') - status = daemon_status[str(daemon_id)][str(fs_id)] + # running a single mirror daemon is supported + status = daemon_status[0] log.debug(f'status: {status}') return status @@ -666,49 +655,53 @@ class TestMirroring(CephFSTestCase): def test_cephfs_mirror_service_daemon_status(self): self.enable_mirroring(self.primary_fs_name, self.primary_fs_id) self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name) - peer_uuid = self.get_peer_uuid("client.mirror_remote@ceph") - - daemon_id = self.get_mirror_daemon_id() time.sleep(30) - status = self.get_mirror_daemon_status(daemon_id, self.primary_fs_name, self.primary_fs_id) + status = self.get_mirror_daemon_status(self.primary_fs_name, self.primary_fs_id) - # we have not added any directories - self.assertEquals(status['directory_count'], 0) + # assumption for this test: mirroring enabled for a single filesystem w/ single + # peer - peer_stats = status['peers'][peer_uuid]['stats'] - self.assertEquals(peer_stats['failure_count'], 0) - self.assertEquals(peer_stats['recovery_count'], 0) + # we have not added any directories + peer = status['filesystems'][0]['peers'][0] + self.assertEquals(status['filesystems'][0]['directory_count'], 0) + self.assertEquals(peer['stats']['failure_count'], 0) + self.assertEquals(peer['stats']['recovery_count'], 0) # add a non-existent directory for synchronization -- check if its reported # in daemon stats self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d0') time.sleep(120) - status = self.get_mirror_daemon_status(daemon_id, self.primary_fs_name, self.primary_fs_id) + status = self.get_mirror_daemon_status(self.primary_fs_name, self.primary_fs_id) # we added one - self.assertEquals(status['directory_count'], 1) - peer_stats = status['peers'][peer_uuid]['stats'] + peer = status['filesystems'][0]['peers'][0] + self.assertEquals(status['filesystems'][0]['directory_count'], 1) # failure count should be reflected - self.assertEquals(peer_stats['failure_count'], 1) - self.assertEquals(peer_stats['recovery_count'], 0) + self.assertEquals(peer['stats']['failure_count'], 1) + self.assertEquals(peer['stats']['recovery_count'], 0) # create the directory, mirror daemon would recover self.mount_a.run_shell(["mkdir", "d0"]) time.sleep(120) - status = self.get_mirror_daemon_status(daemon_id, self.primary_fs_name, self.primary_fs_id) - self.assertEquals(status['directory_count'], 1) - peer_stats = status['peers'][peer_uuid]['stats'] + status = self.get_mirror_daemon_status(self.primary_fs_name, self.primary_fs_id) + peer = status['filesystems'][0]['peers'][0] + self.assertEquals(status['filesystems'][0]['directory_count'], 1) # failure and recovery count should be reflected - self.assertEquals(peer_stats['failure_count'], 1) - self.assertEquals(peer_stats['recovery_count'], 1) + self.assertEquals(peer['stats']['failure_count'], 1) + self.assertEquals(peer['stats']['recovery_count'], 1) self.disable_mirroring(self.primary_fs_name, self.primary_fs_id) def test_mirroring_init_failure(self): """Test mirror daemon init failure""" + # disable mgr mirroring plugin as it would try to load dir map on + # on mirroring enabled for a filesystem (an throw up erorrs in + # the logs) + self.disable_mirroring_module() + # enable mirroring through mon interface -- this should result in the mirror daemon # failing to enable mirroring due to absence of `cephfs_mirorr` index object. self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", "mirror", "enable", self.primary_fs_name) @@ -848,3 +841,328 @@ class TestMirroring(CephFSTestCase): self.remove_directory(self.primary_fs_name, self.primary_fs_id, '/d0') self.disable_mirroring(self.primary_fs_name, self.primary_fs_id) + + def test_cephfs_mirror_with_parent_snapshot(self): + """Test snapshot synchronization with parent directory snapshots""" + self.mount_a.run_shell(["mkdir", "-p", "d0/d1/d2/d3"]) + + self.enable_mirroring(self.primary_fs_name, self.primary_fs_id) + self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d0/d1/d2/d3') + self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name) + + # take a snapshot + self.mount_a.run_shell(["mkdir", "d0/d1/d2/d3/.snap/snap0"]) + + time.sleep(30) + self.check_peer_status(self.primary_fs_name, self.primary_fs_id, + "client.mirror_remote@ceph", '/d0/d1/d2/d3', 'snap0', 1) + + # create snapshots in parent directories + self.mount_a.run_shell(["mkdir", "d0/.snap/snap_d0"]) + self.mount_a.run_shell(["mkdir", "d0/d1/.snap/snap_d1"]) + self.mount_a.run_shell(["mkdir", "d0/d1/d2/.snap/snap_d2"]) + + # try syncing more snapshots + self.mount_a.run_shell(["mkdir", "d0/d1/d2/d3/.snap/snap1"]) + time.sleep(30) + self.check_peer_status(self.primary_fs_name, self.primary_fs_id, + "client.mirror_remote@ceph", '/d0/d1/d2/d3', 'snap1', 2) + + self.mount_a.run_shell(["rmdir", "d0/d1/d2/d3/.snap/snap0"]) + self.mount_a.run_shell(["rmdir", "d0/d1/d2/d3/.snap/snap1"]) + time.sleep(15) + self.check_peer_status_deleted_snap(self.primary_fs_name, self.primary_fs_id, + "client.mirror_remote@ceph", '/d0/d1/d2/d3', 2) + + self.remove_directory(self.primary_fs_name, self.primary_fs_id, '/d0/d1/d2/d3') + self.disable_mirroring(self.primary_fs_name, self.primary_fs_id) + + def test_cephfs_mirror_remove_on_stall(self): + self.enable_mirroring(self.primary_fs_name, self.primary_fs_id) + + # fetch rados address for blacklist check + rados_inst = self.get_mirror_rados_addr(self.primary_fs_name, self.primary_fs_id) + + # simulate non-responding mirror daemon by sending SIGSTOP + pid = self.get_mirror_daemon_pid() + log.debug(f'SIGSTOP to cephfs-mirror pid {pid}') + self.mount_a.run_shell(['kill', '-SIGSTOP', pid]) + + # wait for blocklist timeout -- the manager module would blocklist + # the mirror daemon + time.sleep(40) + + # make sure the rados addr is blocklisted + blocklist = self.get_blocklisted_instances() + self.assertTrue(rados_inst in blocklist) + + # now we are sure that there are no "active" mirror daemons -- add a directory path. + dir_path_p = "/d0/d1" + dir_path = "/d0/d1/d2" + + self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", "snapshot", "mirror", "add", self.primary_fs_name, dir_path) + + time.sleep(10) + # this uses an undocumented interface to get dirpath map state + res_json = self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", "snapshot", "mirror", "dirmap", self.primary_fs_name, dir_path) + res = json.loads(res_json) + # there are no mirror daemons + self.assertTrue(res['state'], 'stalled') + + self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", "snapshot", "mirror", "remove", self.primary_fs_name, dir_path) + + time.sleep(10) + try: + self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", "snapshot", "mirror", "dirmap", self.primary_fs_name, dir_path) + except CommandFailedError as ce: + if ce.exitstatus != errno.ENOENT: + raise RuntimeError('invalid errno when checking dirmap status for non-existent directory') + else: + raise RuntimeError('incorrect errno when checking dirmap state for non-existent directory') + + # adding a parent directory should be allowed + self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", "snapshot", "mirror", "add", self.primary_fs_name, dir_path_p) + + time.sleep(10) + # however, this directory path should get stalled too + res_json = self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", "snapshot", "mirror", "dirmap", self.primary_fs_name, dir_path_p) + res = json.loads(res_json) + # there are no mirror daemons + self.assertTrue(res['state'], 'stalled') + + # wake up the mirror daemon -- at this point, the daemon should know + # that it has been blocklisted + log.debug('SIGCONT to cephfs-mirror') + self.mount_a.run_shell(['kill', '-SIGCONT', pid]) + + # wait for restart mirror on blocklist + time.sleep(60) + res_json = self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", "snapshot", "mirror", "dirmap", self.primary_fs_name, dir_path_p) + res = json.loads(res_json) + # there are no mirror daemons + self.assertTrue(res['state'], 'mapped') + + self.disable_mirroring(self.primary_fs_name, self.primary_fs_id) + + def test_cephfs_mirror_incremental_sync(self): + """ Test incremental snapshot synchronization (based on mtime differences).""" + log.debug('reconfigure client auth caps') + self.mds_cluster.mon_manager.raw_cluster_cmd_result( + 'auth', 'caps', "client.{0}".format(self.mount_b.client_id), + 'mds', 'allow rw', + 'mon', 'allow r', + 'osd', 'allow rw pool={0}, allow rw pool={1}'.format( + self.backup_fs.get_data_pool_name(), self.backup_fs.get_data_pool_name())) + log.debug(f'mounting filesystem {self.secondary_fs_name}') + self.mount_b.umount_wait() + self.mount_b.mount(cephfs_name=self.secondary_fs_name) + + repo = 'ceph-qa-suite' + repo_dir = 'ceph_repo' + repo_path = f'{repo_dir}/{repo}' + + def clone_repo(): + self.mount_a.run_shell([ + 'git', 'clone', '--branch', 'giant', + f'http://github.com/ceph/{repo}', repo_path]) + + def exec_git_cmd(cmd_list): + self.mount_a.run_shell(['git', '--git-dir', f'{self.mount_a.mountpoint}/{repo_path}/.git', *cmd_list]) + + self.mount_a.run_shell(["mkdir", repo_dir]) + clone_repo() + + self.enable_mirroring(self.primary_fs_name, self.primary_fs_id) + self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name) + + self.add_directory(self.primary_fs_name, self.primary_fs_id, f'/{repo_path}') + self.mount_a.run_shell(['mkdir', f'{repo_path}/.snap/snap_a']) + + # full copy, takes time + time.sleep(500) + self.check_peer_status(self.primary_fs_name, self.primary_fs_id, + "client.mirror_remote@ceph", f'/{repo_path}', 'snap_a', 1) + self.verify_snapshot(repo_path, 'snap_a') + + # create some diff + num = random.randint(5, 20) + log.debug(f'resetting to HEAD~{num}') + exec_git_cmd(["reset", "--hard", f'HEAD~{num}']) + + self.mount_a.run_shell(['mkdir', f'{repo_path}/.snap/snap_b']) + # incremental copy, should be fast + time.sleep(180) + self.check_peer_status(self.primary_fs_name, self.primary_fs_id, + "client.mirror_remote@ceph", f'/{repo_path}', 'snap_b', 2) + self.verify_snapshot(repo_path, 'snap_b') + + # diff again, this time back to HEAD + log.debug('resetting to HEAD') + exec_git_cmd(["pull"]) + + self.mount_a.run_shell(['mkdir', f'{repo_path}/.snap/snap_c']) + # incremental copy, should be fast + time.sleep(180) + self.check_peer_status(self.primary_fs_name, self.primary_fs_id, + "client.mirror_remote@ceph", f'/{repo_path}', 'snap_c', 3) + self.verify_snapshot(repo_path, 'snap_c') + + self.disable_mirroring(self.primary_fs_name, self.primary_fs_id) + + def test_cephfs_mirror_incremental_sync_with_type_mixup(self): + """ Test incremental snapshot synchronization with file type changes. + + The same filename exist as a different type in subsequent snapshot. + This verifies if the mirror daemon can identify file type mismatch and + sync snapshots. + + \ snap_0 snap_1 snap_2 snap_3 + \----------------------------------------------- + file_x | reg sym dir reg + | + file_y | dir reg sym dir + | + file_z | sym dir reg sym + """ + log.debug('reconfigure client auth caps') + self.mds_cluster.mon_manager.raw_cluster_cmd_result( + 'auth', 'caps', "client.{0}".format(self.mount_b.client_id), + 'mds', 'allow rw', + 'mon', 'allow r', + 'osd', 'allow rw pool={0}, allow rw pool={1}'.format( + self.backup_fs.get_data_pool_name(), self.backup_fs.get_data_pool_name())) + log.debug(f'mounting filesystem {self.secondary_fs_name}') + self.mount_b.umount_wait() + self.mount_b.mount(cephfs_name=self.secondary_fs_name) + + typs = deque(['reg', 'dir', 'sym']) + def cleanup_and_create_with_type(dirname, fnames): + self.mount_a.run_shell_payload(f"rm -rf {dirname}/*") + fidx = 0 + for t in typs: + fname = f'{dirname}/{fnames[fidx]}' + log.debug(f'file: {fname} type: {t}') + if t == 'reg': + self.mount_a.run_shell(["touch", fname]) + self.mount_a.write_file(fname, data=fname) + elif t == 'dir': + self.mount_a.run_shell(["mkdir", fname]) + elif t == 'sym': + # verify ELOOP in mirror daemon + self.mount_a.run_shell(["ln", "-s", "..", fname]) + fidx += 1 + + def verify_types(dirname, fnames, snap_name): + tidx = 0 + for fname in fnames: + t = self.mount_b.run_shell_payload(f"stat -c %F {dirname}/.snap/{snap_name}/{fname}").stdout.getvalue().strip() + if typs[tidx] == 'reg': + self.assertEquals('regular file', t) + elif typs[tidx] == 'dir': + self.assertEquals('directory', t) + elif typs[tidx] == 'sym': + self.assertEquals('symbolic link', t) + tidx += 1 + + self.enable_mirroring(self.primary_fs_name, self.primary_fs_id) + self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name) + + self.mount_a.run_shell(["mkdir", "d0"]) + self.add_directory(self.primary_fs_name, self.primary_fs_id, '/d0') + + fnames = ['file_x', 'file_y', 'file_z'] + turns = 0 + while turns != len(typs): + snapname = f'snap_{turns}' + cleanup_and_create_with_type('d0', fnames) + self.mount_a.run_shell(['mkdir', f'd0/.snap/{snapname}']) + time.sleep(30) + self.check_peer_status(self.primary_fs_name, self.primary_fs_id, + "client.mirror_remote@ceph", '/d0', snapname, turns+1) + verify_types('d0', fnames, snapname) + # next type + typs.rotate(1) + turns += 1 + + self.disable_mirroring(self.primary_fs_name, self.primary_fs_id) + + def test_cephfs_mirror_sync_with_purged_snapshot(self): + """Test snapshot synchronization in midst of snapshot deletes. + + Deleted the previous snapshot when the mirror daemon is figuring out + incremental differences between current and previous snaphot. The + mirror daemon should identify the purge and switch to using remote + comparison to sync the snapshot (in the next iteration of course). + """ + + log.debug('reconfigure client auth caps') + self.mds_cluster.mon_manager.raw_cluster_cmd_result( + 'auth', 'caps', "client.{0}".format(self.mount_b.client_id), + 'mds', 'allow rw', + 'mon', 'allow r', + 'osd', 'allow rw pool={0}, allow rw pool={1}'.format( + self.backup_fs.get_data_pool_name(), self.backup_fs.get_data_pool_name())) + log.debug(f'mounting filesystem {self.secondary_fs_name}') + self.mount_b.umount_wait() + self.mount_b.mount(cephfs_name=self.secondary_fs_name) + + repo = 'ceph-qa-suite' + repo_dir = 'ceph_repo' + repo_path = f'{repo_dir}/{repo}' + + def clone_repo(): + self.mount_a.run_shell([ + 'git', 'clone', '--branch', 'giant', + f'http://github.com/ceph/{repo}', repo_path]) + + def exec_git_cmd(cmd_list): + self.mount_a.run_shell(['git', '--git-dir', f'{self.mount_a.mountpoint}/{repo_path}/.git', *cmd_list]) + + self.mount_a.run_shell(["mkdir", repo_dir]) + clone_repo() + + self.enable_mirroring(self.primary_fs_name, self.primary_fs_id) + self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name) + + self.add_directory(self.primary_fs_name, self.primary_fs_id, f'/{repo_path}') + self.mount_a.run_shell(['mkdir', f'{repo_path}/.snap/snap_a']) + + # full copy, takes time + time.sleep(500) + self.check_peer_status(self.primary_fs_name, self.primary_fs_id, + "client.mirror_remote@ceph", f'/{repo_path}', 'snap_a', 1) + self.verify_snapshot(repo_path, 'snap_a') + + # create some diff + num = random.randint(60, 100) + log.debug(f'resetting to HEAD~{num}') + exec_git_cmd(["reset", "--hard", f'HEAD~{num}']) + + self.mount_a.run_shell(['mkdir', f'{repo_path}/.snap/snap_b']) + + time.sleep(15) + self.mount_a.run_shell(['rmdir', f'{repo_path}/.snap/snap_a']) + + # incremental copy but based on remote dir_root + time.sleep(300) + self.check_peer_status(self.primary_fs_name, self.primary_fs_id, + "client.mirror_remote@ceph", f'/{repo_path}', 'snap_b', 2) + self.verify_snapshot(repo_path, 'snap_b') + + self.disable_mirroring(self.primary_fs_name, self.primary_fs_id) + + def test_cephfs_mirror_peer_add_primary(self): + self.enable_mirroring(self.primary_fs_name, self.primary_fs_id) + self.peer_add(self.primary_fs_name, self.primary_fs_id, "client.mirror_remote@ceph", self.secondary_fs_name) + + # try adding the primary file system as a peer to secondary file + # system + try: + self.peer_add(self.secondary_fs_name, self.secondary_fs_id, "client.mirror_remote@ceph", self.primary_fs_name) + except CommandFailedError as ce: + if ce.exitstatus != errno.EINVAL: + raise RuntimeError('invalid errno when adding a primary file system') + else: + raise RuntimeError('adding peer should fail') + + self.disable_mirroring(self.primary_fs_name, self.primary_fs_id) diff --git a/ceph/qa/tasks/cephfs/test_multimds_misc.py b/ceph/qa/tasks/cephfs/test_multimds_misc.py index 77f15ec2d..3c464e91d 100644 --- a/ceph/qa/tasks/cephfs/test_multimds_misc.py +++ b/ceph/qa/tasks/cephfs/test_multimds_misc.py @@ -97,7 +97,7 @@ class TestScrub2(CephFSTestCase): file_obj_name = "{0:x}.00000000".format(ino) self.fs.radosm(["rmxattr", file_obj_name, "parent"]) - out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive", "force"], 0) + out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive,force"], 0) self.assertNotEqual(out_json, None) self.assertEqual(out_json["return_code"], 0) self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) @@ -140,7 +140,7 @@ class TestScrub2(CephFSTestCase): file_obj_name = "{0:x}.00000000".format(ino) self.fs.radosm(["rmxattr", file_obj_name, "parent"]) - out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive", "force"], 0) + out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive,force"], 0) self.assertNotEqual(out_json, None) res = self.fs.run_scrub(["abort"]) @@ -163,7 +163,7 @@ class TestScrub2(CephFSTestCase): file_obj_name = "{0:x}.00000000".format(ino) self.fs.radosm(["rmxattr", file_obj_name, "parent"]) - out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive", "force"], 0) + out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive,force"], 0) self.assertNotEqual(out_json, None) res = self.fs.run_scrub(["pause"]) @@ -192,7 +192,7 @@ class TestScrub2(CephFSTestCase): file_obj_name = "{0:x}.00000000".format(ino) self.fs.radosm(["rmxattr", file_obj_name, "parent"]) - out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive", "force"], 0) + out_json = self.fs.run_scrub(["start", "/d1/d2/d3", "recursive,force"], 0) self.assertNotEqual(out_json, None) res = self.fs.run_scrub(["pause"]) diff --git a/ceph/qa/tasks/cephfs/test_nfs.py b/ceph/qa/tasks/cephfs/test_nfs.py index 30d9d6df4..83c4797b2 100644 --- a/ceph/qa/tasks/cephfs/test_nfs.py +++ b/ceph/qa/tasks/cephfs/test_nfs.py @@ -31,6 +31,7 @@ class TestNFS(MgrTestCase): def setUp(self): super(TestNFS, self).setUp() + self._load_module('nfs') self.cluster_id = "test" self.export_type = "cephfs" self.pseudo_path = "/cephfs" @@ -118,7 +119,7 @@ class TestNFS(MgrTestCase): ''' # Disable any running nfs ganesha daemon self._check_nfs_server_status() - self._nfs_cmd('cluster', 'create', self.export_type, self.cluster_id) + self._nfs_cmd('cluster', 'create', self.cluster_id) # Check for expected status and daemon name (nfs.) self._check_nfs_cluster_status('running', 'NFS Ganesha cluster deployment failed') @@ -126,7 +127,7 @@ class TestNFS(MgrTestCase): ''' Test deletion of a single nfs cluster. ''' - self._nfs_cmd('cluster', 'delete', self.cluster_id) + self._nfs_cmd('cluster', 'rm', self.cluster_id) self._check_nfs_cluster_status('No daemons reported', 'NFS Ganesha cluster could not be deleted') @@ -178,7 +179,7 @@ class TestNFS(MgrTestCase): ''' Delete an export. ''' - self._nfs_cmd('export', 'delete', self.cluster_id, self.pseudo_path) + self._nfs_cmd('export', 'rm', self.cluster_id, self.pseudo_path) self._check_auth_ls() def _test_list_export(self): @@ -242,9 +243,9 @@ class TestNFS(MgrTestCase): ''' Return port and ip for a cluster ''' - #{'test': [{'hostname': 'smithi068', 'ip': ['172.21.15.68'], 'port': 2049}]} - info_output = json.loads(self._nfs_cmd('cluster', 'info', self.cluster_id))['test'][0] - return info_output["port"], info_output["ip"][0] + #{'test': {'backend': [{'hostname': 'smithi068', 'ip': '172.21.15.68', 'port': 2049}]}} + info_output = json.loads(self._nfs_cmd('cluster', 'info', self.cluster_id))['test']['backend'][0] + return info_output["port"], info_output["ip"] def _test_mnt(self, pseudo_path, port, ip, check=True): ''' @@ -295,9 +296,8 @@ class TestNFS(MgrTestCase): ''' Test idempotency of cluster create and delete commands. ''' - self._test_idempotency(self._test_create_cluster, ['nfs', 'cluster', 'create', self.export_type, - self.cluster_id]) - self._test_idempotency(self._test_delete_cluster, ['nfs', 'cluster', 'delete', self.cluster_id]) + self._test_idempotency(self._test_create_cluster, ['nfs', 'cluster', 'create', self.cluster_id]) + self._test_idempotency(self._test_delete_cluster, ['nfs', 'cluster', 'rm', self.cluster_id]) def test_create_cluster_with_invalid_cluster_id(self): ''' @@ -305,26 +305,13 @@ class TestNFS(MgrTestCase): ''' try: invalid_cluster_id = '/cluster_test' # Only [A-Za-z0-9-_.] chars are valid - self._nfs_cmd('cluster', 'create', self.export_type, invalid_cluster_id) + self._nfs_cmd('cluster', 'create', invalid_cluster_id) self.fail(f"Cluster successfully created with invalid cluster id {invalid_cluster_id}") except CommandFailedError as e: # Command should fail for test to pass if e.exitstatus != errno.EINVAL: raise - def test_create_cluster_with_invalid_export_type(self): - ''' - Test nfs cluster deployment failure with invalid export type. - ''' - try: - invalid_export_type = 'rgw' # Only cephfs is valid - self._nfs_cmd('cluster', 'create', invalid_export_type, self.cluster_id) - self.fail(f"Cluster successfully created with invalid export type {invalid_export_type}") - except CommandFailedError as e: - # Command should fail for test to pass - if e.exitstatus != errno.EINVAL: - raise - def test_create_and_delete_export(self): ''' Test successful creation and deletion of the cephfs export. @@ -346,7 +333,7 @@ class TestNFS(MgrTestCase): self._test_idempotency(self._create_default_export, ['nfs', 'export', 'create', 'cephfs', self.fs_name, self.cluster_id, self.pseudo_path]) - self._test_idempotency(self._delete_export, ['nfs', 'export', 'delete', self.cluster_id, + self._test_idempotency(self._delete_export, ['nfs', 'export', 'rm', self.cluster_id, self.pseudo_path]) self._test_delete_cluster() @@ -456,14 +443,23 @@ class TestNFS(MgrTestCase): ''' self._test_create_cluster() info_output = json.loads(self._nfs_cmd('cluster', 'info', self.cluster_id)) - info_ip = info_output[self.cluster_id][0].pop("ip") - host_details = {self.cluster_id: [{ - "hostname": self._sys_cmd(['hostname']).decode("utf-8").strip(), - "port": 2049 - }]} + print(f'info {info_output}') + info_ip = info_output[self.cluster_id].get('backend', [])[0].pop("ip") + host_details = { + self.cluster_id: { + 'backend': [ + { + "hostname": self._sys_cmd(['hostname']).decode("utf-8").strip(), + "port": 2049 + } + ], + "virtual_ip": None, + } + } host_ip = self._sys_cmd(['hostname', '-I']).decode("utf-8").split() + print(f'host_ip is {host_ip}, info_ip is {info_ip}') self.assertDictEqual(info_output, host_details) - self.assertTrue(any([ip in info_ip for ip in host_ip])) + self.assertTrue(info_ip in host_ip) self._test_delete_cluster() def test_cluster_set_reset_user_config(self): @@ -596,3 +592,29 @@ class TestNFS(MgrTestCase): update_with_invalid_values('user_id', 'testing_export', True) update_with_invalid_values('fs_name', 'b', True) self._test_delete_cluster() + + def test_cmds_without_reqd_args(self): + ''' + Test that cmd fails on not passing required arguments + ''' + def exec_cmd_invalid(*cmd): + try: + self._nfs_cmd(*cmd) + self.fail(f"nfs {cmd} command executed successfully without required arguments") + except CommandFailedError as e: + # Command should fail for test to pass + if e.exitstatus != errno.EINVAL: + raise + + exec_cmd_invalid('cluster', 'create') + exec_cmd_invalid('cluster', 'delete') + exec_cmd_invalid('cluster', 'config', 'set') + exec_cmd_invalid('cluster', 'config', 'reset') + exec_cmd_invalid('export', 'create', 'cephfs') + exec_cmd_invalid('export', 'create', 'cephfs', 'a_fs') + exec_cmd_invalid('export', 'create', 'cephfs', 'a_fs', 'clusterid') + exec_cmd_invalid('export', 'ls') + exec_cmd_invalid('export', 'delete') + exec_cmd_invalid('export', 'delete', 'clusterid') + exec_cmd_invalid('export', 'get') + exec_cmd_invalid('export', 'get', 'clusterid') diff --git a/ceph/qa/tasks/cephfs/test_recovery_pool.py b/ceph/qa/tasks/cephfs/test_recovery_pool.py index 9742bfa47..9926b3670 100644 --- a/ceph/qa/tasks/cephfs/test_recovery_pool.py +++ b/ceph/qa/tasks/cephfs/test_recovery_pool.py @@ -180,7 +180,7 @@ class TestRecoveryPool(CephFSTestCase): for rank in self.recovery_fs.get_ranks(status=status): self.fs.mon_manager.raw_cluster_cmd('tell', "mds." + rank['name'], 'injectargs', '--debug-mds=20') - self.fs.rank_tell(['scrub', 'start', '/', 'recursive', 'repair'], rank=rank['rank'], status=status) + self.fs.rank_tell(['scrub', 'start', '/', 'recursive,repair'], rank=rank['rank'], status=status) log.info(str(self.mds_cluster.status())) # Mount a client diff --git a/ceph/qa/tasks/cephfs/test_scrub.py b/ceph/qa/tasks/cephfs/test_scrub.py index 5d571460a..dd7c11af5 100644 --- a/ceph/qa/tasks/cephfs/test_scrub.py +++ b/ceph/qa/tasks/cephfs/test_scrub.py @@ -103,7 +103,7 @@ class DupInodeWorkload(Workload): self._filesystem.wait_for_daemons() def validate(self): - out_json = self._filesystem.run_scrub(["start", "/", "recursive", "repair"]) + out_json = self._filesystem.run_scrub(["start", "/", "recursive,repair"]) self.assertNotEqual(out_json, None) self.assertEqual(out_json["return_code"], 0) self.assertEqual(self._filesystem.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) @@ -134,7 +134,7 @@ class TestScrub(CephFSTestCase): # Apply any data damage the workload wants workload.damage() - out_json = self.fs.run_scrub(["start", "/", "recursive", "repair"]) + out_json = self.fs.run_scrub(["start", "/", "recursive,repair"]) self.assertNotEqual(out_json, None) self.assertEqual(out_json["return_code"], 0) self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True) diff --git a/ceph/qa/tasks/cephfs/test_snap_schedules.py b/ceph/qa/tasks/cephfs/test_snap_schedules.py index f0e5a3423..1f65106c8 100644 --- a/ceph/qa/tasks/cephfs/test_snap_schedules.py +++ b/ceph/qa/tasks/cephfs/test_snap_schedules.py @@ -38,9 +38,11 @@ class TestSnapSchedules(CephFSTestCase): def _fs_cmd(self, *args): return self.mgr_cluster.mon_manager.raw_cluster_cmd("fs", *args) - def fs_snap_schedule_cmd(self, *args): - args = list(args) - args.append(f'fs={self.volname}') + def fs_snap_schedule_cmd(self, *args, **kwargs): + fs = kwargs.pop('fs', self.volname) + args += ('--fs', fs) + for name, val in kwargs.items(): + args += (f'--{name}', str(val)) res = self._fs_cmd('snap-schedule', *args) log.debug(f'res={res}') return res @@ -144,7 +146,7 @@ class TestSnapSchedules(CephFSTestCase): def verify_schedule(self, dir_path, schedules, retentions=[]): log.debug(f'expected_schedule: {schedules}, expected_retention: {retentions}') - result = self.fs_snap_schedule_cmd('list', f'path={dir_path}', 'format=json') + result = self.fs_snap_schedule_cmd('list', path=dir_path, format='json') json_res = json.loads(result) log.debug(f'json_res: {json_res}') @@ -165,7 +167,7 @@ class TestSnapSchedules(CephFSTestCase): def test_non_existent_snap_schedule_list(self): """Test listing snap schedules on a non-existing filesystem path failure""" try: - self.fs_snap_schedule_cmd('list', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'format=json') + self.fs_snap_schedule_cmd('list', path=TestSnapSchedules.TEST_DIRECTORY) except CommandFailedError as ce: if ce.exitstatus != errno.ENOENT: raise RuntimeError('incorrect errno when listing a non-existing snap schedule') @@ -177,7 +179,7 @@ class TestSnapSchedules(CephFSTestCase): self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY]) try: - self.fs_snap_schedule_cmd('list', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'format=json') + self.fs_snap_schedule_cmd('list', path=TestSnapSchedules.TEST_DIRECTORY) except CommandFailedError as ce: if ce.exitstatus != errno.ENOENT: raise RuntimeError('incorrect errno when listing a non-existing snap schedule') @@ -190,12 +192,12 @@ class TestSnapSchedules(CephFSTestCase): """Test listing snap schedules post removal of a schedule""" self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY]) - self.fs_snap_schedule_cmd('add', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'snap-schedule=1h') + self.fs_snap_schedule_cmd('add', path=TestSnapSchedules.TEST_DIRECTORY, snap_schedule='1h') - self.fs_snap_schedule_cmd('remove', f'path={TestSnapSchedules.TEST_DIRECTORY}') + self.fs_snap_schedule_cmd('remove', path=TestSnapSchedules.TEST_DIRECTORY) try: - self.fs_snap_schedule_cmd('list', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'format=json') + self.fs_snap_schedule_cmd('list', path=TestSnapSchedules.TEST_DIRECTORY) except CommandFailedError as ce: if ce.exitstatus != errno.ENOENT: raise RuntimeError('incorrect errno when listing a non-existing snap schedule') @@ -209,7 +211,7 @@ class TestSnapSchedules(CephFSTestCase): self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY]) # set a schedule on the dir - self.fs_snap_schedule_cmd('add', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'snap-schedule=1M') + self.fs_snap_schedule_cmd('add', path=TestSnapSchedules.TEST_DIRECTORY, snap_schedule='1M') exec_time = time.time() timo, snap_sfx = self.calc_wait_time_and_snap_name(exec_time, '1M') @@ -232,7 +234,7 @@ class TestSnapSchedules(CephFSTestCase): self.assert_if_not_verified() # remove snapshot schedule - self.fs_snap_schedule_cmd('remove', f'path={TestSnapSchedules.TEST_DIRECTORY}') + self.fs_snap_schedule_cmd('remove', path=TestSnapSchedules.TEST_DIRECTORY) # remove all scheduled snapshots self.remove_snapshots(TestSnapSchedules.TEST_DIRECTORY) @@ -244,8 +246,8 @@ class TestSnapSchedules(CephFSTestCase): self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY]) # set schedules on the dir - self.fs_snap_schedule_cmd('add', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'snap-schedule=1M') - self.fs_snap_schedule_cmd('add', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'snap-schedule=2M') + self.fs_snap_schedule_cmd('add', path=TestSnapSchedules.TEST_DIRECTORY, snap_schedule='1M') + self.fs_snap_schedule_cmd('add', path=TestSnapSchedules.TEST_DIRECTORY, snap_schedule='2M') exec_time = time.time() timo_1, snap_sfx_1 = self.calc_wait_time_and_snap_name(exec_time, '1M') @@ -279,7 +281,7 @@ class TestSnapSchedules(CephFSTestCase): self.assert_if_not_verified() # remove snapshot schedule - self.fs_snap_schedule_cmd('remove', f'path={TestSnapSchedules.TEST_DIRECTORY}') + self.fs_snap_schedule_cmd('remove', path=TestSnapSchedules.TEST_DIRECTORY) # remove all scheduled snapshots self.remove_snapshots(TestSnapSchedules.TEST_DIRECTORY) @@ -291,8 +293,8 @@ class TestSnapSchedules(CephFSTestCase): self.mount_a.run_shell(['mkdir', '-p', TestSnapSchedules.TEST_DIRECTORY]) # set a schedule on the dir - self.fs_snap_schedule_cmd('add', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'snap-schedule=1M') - self.fs_snap_schedule_cmd('retention', 'add', f'path={TestSnapSchedules.TEST_DIRECTORY}', 'retention-spec-or-period=1M') + self.fs_snap_schedule_cmd('add', path=TestSnapSchedules.TEST_DIRECTORY, snap_schedule='1M') + self.fs_snap_schedule_cmd('retention', 'add', path=TestSnapSchedules.TEST_DIRECTORY, retention_spec_or_period='1M') exec_time = time.time() timo_1, snap_sfx = self.calc_wait_time_and_snap_name(exec_time, '1M') @@ -330,7 +332,7 @@ class TestSnapSchedules(CephFSTestCase): self.assert_if_not_verified() # remove snapshot schedule - self.fs_snap_schedule_cmd('remove', f'path={TestSnapSchedules.TEST_DIRECTORY}') + self.fs_snap_schedule_cmd('remove', path=TestSnapSchedules.TEST_DIRECTORY) # remove all scheduled snapshots self.remove_snapshots(TestSnapSchedules.TEST_DIRECTORY) diff --git a/ceph/qa/tasks/cephfs/test_volume_client.py b/ceph/qa/tasks/cephfs/test_volume_client.py index 0f01d1bb2..d1b2e760c 100644 --- a/ceph/qa/tasks/cephfs/test_volume_client.py +++ b/ceph/qa/tasks/cephfs/test_volume_client.py @@ -1,3 +1,4 @@ +from io import StringIO import json import logging import os @@ -251,7 +252,7 @@ vc.disconnect() ns_in_attr = self.mount_a.getfattr(os.path.join("myprefix", group_id, volume_id), "ceph.dir.layout.pool_namespace") self.assertEqual(namespace, ns_in_attr) - objects_in_ns = set(self.fs.rados(["ls"], pool=pool_name, namespace=namespace).split("\n")) + objects_in_ns = set(self.fs.rados(["ls"], pool=pool_name, namespace=namespace, stdout=StringIO()).stdout.getvalue().split("\n")) self.assertNotEqual(objects_in_ns, set()) # De-authorize the guest @@ -1541,7 +1542,7 @@ vc.disconnect() obj_data = obj_data ))) - read_data = self.fs.rados(['get', obj_name, '-'], pool=pool_name) + read_data = self.fs.rados(['get', obj_name, '-'], pool=pool_name, stdout=StringIO()).stdout.getvalue() self.assertEqual(obj_data, read_data) def test_get_object(self): @@ -1553,7 +1554,7 @@ vc.disconnect() obj_name = 'test_vc_ob_2' pool_name = self.fs.get_data_pool_names()[0] - self.fs.rados(['put', obj_name, '-'], pool=pool_name, stdin_data=obj_data) + self.fs.rados(['put', obj_name, '-'], pool=pool_name, stdin=StringIO(obj_data)) self._volume_client_python(vc_mount, dedent(""" data_read = vc.get_object("{pool_name}", "{obj_name}") @@ -1572,7 +1573,7 @@ vc.disconnect() obj_data = 'test_data' obj_name = 'test_vc_obj' pool_name = self.fs.get_data_pool_names()[0] - self.fs.rados(['put', obj_name, '-'], pool=pool_name, stdin_data=obj_data) + self.fs.rados(['put', obj_name, '-'], pool=pool_name, stdin=StringIO(obj_data)) self._volume_client_python(vc_mount, dedent(""" data, version_before = vc.get_object_and_version("{pool_name}", "{obj_name}") @@ -1595,7 +1596,7 @@ vc.disconnect() obj_data = 'test_data' obj_name = 'test_vc_ob_2' pool_name = self.fs.get_data_pool_names()[0] - self.fs.rados(['put', obj_name, '-'], pool=pool_name, stdin_data=obj_data) + self.fs.rados(['put', obj_name, '-'], pool=pool_name, stdin=StringIO(obj_data)) # Test if put_object_versioned() crosschecks the version of the # given object. Being a negative test, an exception is expected. @@ -1633,7 +1634,7 @@ vc.disconnect() obj_name = 'test_vc_obj_3' pool_name = self.fs.get_data_pool_names()[0] - self.fs.rados(['put', obj_name, '-'], pool=pool_name, stdin_data=obj_data) + self.fs.rados(['put', obj_name, '-'], pool=pool_name, stdin=StringIO(obj_data)) self._volume_client_python(vc_mount, dedent(""" data_read = vc.delete_object("{pool_name}", "{obj_name}") diff --git a/ceph/qa/tasks/daemonwatchdog.py b/ceph/qa/tasks/daemonwatchdog.py index f72ccd7ce..c8fa9f3c2 100644 --- a/ceph/qa/tasks/daemonwatchdog.py +++ b/ceph/qa/tasks/daemonwatchdog.py @@ -55,11 +55,12 @@ class DaemonWatchdog(Greenlet): def bark(self): self.log("BARK! unmounting mounts and killing all daemons") - for mount in self.ctx.mounts.values(): - try: - mount.umount_wait(force=True) - except: - self.logger.exception("ignoring exception:") + if hasattr(self.ctx, 'mounts'): + for mount in self.ctx.mounts.values(): + try: + mount.umount_wait(force=True) + except: + self.logger.exception("ignoring exception:") daemons = [] daemons.extend(filter(lambda daemon: daemon.running() and not daemon.proc.finished, self.ctx.daemons.iter_daemons_of_role('osd', cluster=self.cluster))) daemons.extend(filter(lambda daemon: daemon.running() and not daemon.proc.finished, self.ctx.daemons.iter_daemons_of_role('mds', cluster=self.cluster))) diff --git a/ceph/qa/tasks/fwd_scrub.py b/ceph/qa/tasks/fwd_scrub.py index a7ae1c4ed..44fd97baa 100644 --- a/ceph/qa/tasks/fwd_scrub.py +++ b/ceph/qa/tasks/fwd_scrub.py @@ -61,8 +61,10 @@ class ForwardScrubber(Thrasher, Greenlet): def _scrub(self, path="/", recursive=True): self.logger.info(f"scrubbing fs: {self.fs.name}") - recopt = ["recursive", "force"] if recursive else ["force"] - out_json = self.fs.run_scrub(["start", path] + recopt) + scrubopts = ["force"] + if recursive: + scrubopts.append("recursive") + out_json = self.fs.run_scrub(["start", path, ",".join(scrubopts)]) assert out_json is not None tag = out_json['scrub_tag'] diff --git a/ceph/qa/tasks/kclient.py b/ceph/qa/tasks/kclient.py index be75286bd..d7bc9fa83 100644 --- a/ceph/qa/tasks/kclient.py +++ b/ceph/qa/tasks/kclient.py @@ -84,9 +84,7 @@ def task(ctx, config): if client_config is None: client_config = {} # top level overrides - for k, v in top_overrides.items(): - if v is not None: - client_config[k] = v + deep_merge(client_config, top_overrides) # mount specific overrides client_config_overrides = overrides.get(entity) deep_merge(client_config, client_config_overrides) diff --git a/ceph/qa/tasks/kubeadm.py b/ceph/qa/tasks/kubeadm.py new file mode 100644 index 000000000..c870bbae8 --- /dev/null +++ b/ceph/qa/tasks/kubeadm.py @@ -0,0 +1,536 @@ +""" +Kubernetes cluster task, deployed via kubeadm +""" +import argparse +import contextlib +import ipaddress +import logging +import random +import yaml +from io import BytesIO + +from teuthology import misc as teuthology +from teuthology import contextutil +from teuthology.config import config as teuth_config +from teuthology.orchestra import run + +log = logging.getLogger(__name__) + + +def _kubectl(ctx, config, args, **kwargs): + cluster_name = config['cluster'] + ctx.kubeadm[cluster_name].bootstrap_remote.run( + args=['kubectl'] + args, + **kwargs, + ) + + +def kubectl(ctx, config): + if isinstance(config, str): + config = [config] + assert isinstance(config, list) + for c in config: + if isinstance(c, str): + _kubectl(ctx, config, c.split(' ')) + else: + _kubectl(ctx, config, c) + + +@contextlib.contextmanager +def preflight(ctx, config): + run.wait( + ctx.cluster.run( + args=[ + 'sudo', 'modprobe', 'br_netfilter', + run.Raw('&&'), + 'sudo', 'sysctl', 'net.bridge.bridge-nf-call-ip6tables=1', + run.Raw('&&'), + 'sudo', 'sysctl', 'net.bridge.bridge-nf-call-iptables=1', + run.Raw('&&'), + 'sudo', 'sysctl', 'net.ipv4.ip_forward=1', + run.Raw('&&'), + 'sudo', 'swapoff', '-a', + ], + wait=False, + ) + ) + yield + + +@contextlib.contextmanager +def kubeadm_install(ctx, config): + version = config.get('version', '1.21') + + os_type = teuthology.get_distro(ctx) + os_version = teuthology.get_distro_version(ctx) + + try: + if os_type in ['centos', 'rhel']: + os = f"CentOS_{os_version.split('.')[0]}" + log.info('Installing cri-o') + run.wait( + ctx.cluster.run( + args=[ + 'sudo', + 'curl', '-L', '-o', + '/etc/yum.repos.d/devel:kubic:libcontainers:stable.repo', + f'https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{os}/devel:kubic:libcontainers:stable.repo', + run.Raw('&&'), + 'sudo', + 'curl', '-L', '-o', + f'/etc/yum.repos.d/devel:kubic:libcontainers:stable:cri-o:{version}.repo', + f'https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{version}/{os}/devel:kubic:libcontainers:stable:cri-o:{version}.repo', + run.Raw('&&'), + 'sudo', 'dnf', 'install', '-y', 'cri-o', + ], + wait=False, + ) + ) + + log.info('Installing kube{adm,ctl,let}') + repo = """[kubernetes] +name=Kubernetes +baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-$basearch +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg +""" + for remote in ctx.cluster.remotes.keys(): + remote.write_file( + '/etc/yum.repos.d/kubernetes.repo', + repo, + sudo=True, + ) + run.wait( + ctx.cluster.run( + args=[ + 'sudo', 'dnf', 'install', '-y', + 'kubelet', 'kubeadm', 'kubectl', + 'iproute-tc', 'bridge-utils', + ], + wait=False, + ) + ) + + # fix cni config + for remote in ctx.cluster.remotes.keys(): + conf = """# from https://github.com/cri-o/cri-o/blob/master/tutorials/kubernetes.md#flannel-network +{ + "name": "crio", + "type": "flannel" +} +""" + remote.write_file('/etc/cni/net.d/10-crio-flannel.conf', conf, sudo=True) + remote.run(args=[ + 'sudo', 'rm', '-f', + '/etc/cni/net.d/87-podman-bridge.conflist', + '/etc/cni/net.d/100-crio-bridge.conf', + ]) + + # start crio + run.wait( + ctx.cluster.run( + args=[ + 'sudo', 'systemctl', 'daemon-reload', + run.Raw('&&'), + 'sudo', 'systemctl', 'enable', 'crio', '--now', + ], + wait=False, + ) + ) + + elif os_type == 'ubuntu': + os = f"xUbuntu_{os_version}" + log.info('Installing kube{adm,ctl,let}') + run.wait( + ctx.cluster.run( + args=[ + 'sudo', 'apt', 'update', + run.Raw('&&'), + 'sudo', 'apt', 'install', '-y', + 'apt-transport-https', 'ca-certificates', 'curl', + run.Raw('&&'), + 'sudo', 'curl', '-fsSLo', + '/usr/share/keyrings/kubernetes-archive-keyring.gpg', + 'https://packages.cloud.google.com/apt/doc/apt-key.gpg', + run.Raw('&&'), + 'echo', 'deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main', + run.Raw('|'), + 'sudo', 'tee', '/etc/apt/sources.list.d/kubernetes.list', + run.Raw('&&'), + 'sudo', 'apt', 'update', + run.Raw('&&'), + 'sudo', 'apt', 'install', '-y', + 'kubelet', 'kubeadm', 'kubectl', + 'bridge-utils', + ], + wait=False, + ) + ) + + else: + raise RuntimeError(f'unsupported distro {os_type} for cri-o') + + run.wait( + ctx.cluster.run( + args=[ + 'sudo', 'systemctl', 'enable', '--now', 'kubelet', + run.Raw('&&'), + 'sudo', 'kubeadm', 'config', 'images', 'pull', + ], + wait=False, + ) + ) + + yield + + finally: + if config.get('uninstall', True): + log.info('Uninstalling kube{adm,let,ctl}') + if os_type in ['centos', 'rhel']: + run.wait( + ctx.cluster.run( + args=[ + 'sudo', 'rm', '-f', + '/etc/yum.repos.d/kubernetes.repo', + run.Raw('&&'), + 'sudo', 'dnf', 'remove', '-y', + 'kubeadm', 'kubelet', 'kubectl', 'cri-o', + ], + wait=False + ) + ) + elif os_type == 'ubuntu' and False: + run.wait( + ctx.cluster.run( + args=[ + 'sudo', 'rm', '-f', + '/etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list', + f'/etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:{version}.list', + '/etc/apt/trusted.gpg.d/libcontainers-cri-o.gpg', + run.Raw('&&'), + 'sudo', 'apt', 'remove', '-y', + 'kkubeadm', 'kubelet', 'kubectl', 'cri-o', 'cri-o-runc', + ], + wait=False, + ) + ) + + +@contextlib.contextmanager +def kubeadm_init_join(ctx, config): + cluster_name = config['cluster'] + + bootstrap_remote = None + remotes = {} # remote -> ip + for remote, roles in ctx.cluster.remotes.items(): + for role in roles: + if role.startswith('host.'): + if not bootstrap_remote: + bootstrap_remote = remote + if remote not in remotes: + remotes[remote] = remote.ssh.get_transport().getpeername()[0] + if not bootstrap_remote: + raise RuntimeError('must define at least one host.something role') + ctx.kubeadm[cluster_name].bootstrap_remote = bootstrap_remote + ctx.kubeadm[cluster_name].remotes = remotes + ctx.kubeadm[cluster_name].token = 'abcdef.' + ''.join([ + random.choice('0123456789abcdefghijklmnopqrstuvwxyz') for _ in range(16) + ]) + log.info(f'Token: {ctx.kubeadm[cluster_name].token}') + log.info(f'Remotes: {ctx.kubeadm[cluster_name].remotes}') + + try: + # init + cmd = [ + 'sudo', 'kubeadm', 'init', + '--node-name', ctx.kubeadm[cluster_name].bootstrap_remote.shortname, + '--token', ctx.kubeadm[cluster_name].token, + '--pod-network-cidr', str(ctx.kubeadm[cluster_name].pod_subnet), + ] + bootstrap_remote.run(args=cmd) + + # join additional nodes + joins = [] + for remote, ip in ctx.kubeadm[cluster_name].remotes.items(): + if remote == bootstrap_remote: + continue + cmd = [ + 'sudo', 'kubeadm', 'join', + ctx.kubeadm[cluster_name].remotes[ctx.kubeadm[cluster_name].bootstrap_remote] + ':6443', + '--node-name', remote.shortname, + '--token', ctx.kubeadm[cluster_name].token, + '--discovery-token-unsafe-skip-ca-verification', + ] + joins.append(remote.run(args=cmd, wait=False)) + run.wait(joins) + yield + + except Exception as e: + log.exception(e) + raise + + finally: + log.info('Cleaning up node') + run.wait( + ctx.cluster.run( + args=['sudo', 'kubeadm', 'reset', 'cleanup-node', '-f'], + wait=False, + ) + ) + + +@contextlib.contextmanager +def kubectl_config(ctx, config): + cluster_name = config['cluster'] + bootstrap_remote = ctx.kubeadm[cluster_name].bootstrap_remote + + ctx.kubeadm[cluster_name].admin_conf = \ + bootstrap_remote.read_file('/etc/kubernetes/admin.conf', sudo=True) + + log.info('Setting up kubectl') + try: + ctx.cluster.run(args=[ + 'mkdir', '-p', '.kube', + run.Raw('&&'), + 'sudo', 'mkdir', '-p', '/root/.kube', + ]) + for remote in ctx.kubeadm[cluster_name].remotes.keys(): + remote.write_file('.kube/config', ctx.kubeadm[cluster_name].admin_conf) + remote.sudo_write_file('/root/.kube/config', + ctx.kubeadm[cluster_name].admin_conf) + yield + + except Exception as e: + log.exception(e) + raise + + finally: + log.info('Deconfiguring kubectl') + ctx.cluster.run(args=[ + 'rm', '-rf', '.kube', + run.Raw('&&'), + 'sudo', 'rm', '-rf', '/root/.kube', + ]) + + +def map_vnet(mip): + for mapping in teuth_config.get('vnet', []): + mnet = ipaddress.ip_network(mapping['machine_subnet']) + vnet = ipaddress.ip_network(mapping['virtual_subnet']) + if vnet.prefixlen >= mnet.prefixlen: + log.error(f"virtual_subnet {vnet} prefix >= machine_subnet {mnet} prefix") + return None + if mip in mnet: + pos = list(mnet.hosts()).index(mip) + log.info(f"{mip} is in {mnet} at pos {pos}") + sub = list(vnet.subnets(32 - mnet.prefixlen))[pos] + return sub + return None + + +@contextlib.contextmanager +def allocate_pod_subnet(ctx, config): + """ + Allocate a private subnet that will not collide with other test machines/clusters + """ + cluster_name = config['cluster'] + assert cluster_name == 'kubeadm', 'multiple subnets not yet implemented' + + log.info('Identifying pod subnet') + remote = list(ctx.cluster.remotes.keys())[0] + ip = remote.ssh.get_transport().getpeername()[0] + mip = ipaddress.ip_address(ip) + vnet = map_vnet(mip) + assert vnet + log.info(f'Pod subnet: {vnet}') + ctx.kubeadm[cluster_name].pod_subnet = vnet + yield + + +@contextlib.contextmanager +def pod_network(ctx, config): + cluster_name = config['cluster'] + pnet = config.get('pod_network', 'calico') + if pnet == 'flannel': + r = ctx.kubeadm[cluster_name].bootstrap_remote.run( + args=[ + 'curl', + 'https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml', + ], + stdout=BytesIO(), + ) + assert r.exitstatus == 0 + flannel = list(yaml.load_all(r.stdout.getvalue(), Loader=yaml.FullLoader)) + for o in flannel: + if o.get('data', {}).get('net-conf.json'): + log.info(f'Updating {o}') + o['data']['net-conf.json'] = o['data']['net-conf.json'].replace( + '10.244.0.0/16', + str(ctx.kubeadm[cluster_name].pod_subnet) + ) + log.info(f'Now {o}') + flannel_yaml = yaml.dump_all(flannel) + log.debug(f'Flannel:\n{flannel_yaml}') + _kubectl(ctx, config, ['apply', '-f', '-'], stdin=flannel_yaml) + + elif pnet == 'calico': + _kubectl(ctx, config, [ + 'apply', '-f', + 'https://docs.projectcalico.org/manifests/tigera-operator.yaml' + ]) + cr = { + 'apiVersion': 'operator.tigera.io/v1', + 'kind': 'Installation', + 'metadata': {'name': 'default'}, + 'spec': { + 'calicoNetwork': { + 'ipPools': [ + { + 'blockSize': 26, + 'cidr': str(ctx.kubeadm[cluster_name].pod_subnet), + 'encapsulation': 'VXLANCrossSubnet', + 'natOutgoing': 'Enabled', + 'nodeSelector': 'all()', + } + ] + } + } + } + _kubectl(ctx, config, ['create', '-f', '-'], stdin=yaml.dump(cr)) + + else: + raise RuntimeError(f'unrecognized pod_network {pnet}') + + try: + yield + + finally: + if pnet == 'flannel': + _kubectl(ctx, config, [ + 'delete', '-f', + 'https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml', + ]) + + elif pnet == 'calico': + _kubectl(ctx, config, ['delete', 'installation', 'default']) + _kubectl(ctx, config, [ + 'delete', '-f', + 'https://docs.projectcalico.org/manifests/tigera-operator.yaml' + ]) + + +@contextlib.contextmanager +def setup_pvs(ctx, config): + """ + Create PVs for all scratch LVs and set up a trivial provisioner + """ + log.info('Scanning for scratch devices') + crs = [] + for remote in ctx.cluster.remotes.keys(): + ls = remote.read_file('/scratch_devs').decode('utf-8').strip().splitlines() + log.info(f'Scratch devices on {remote.shortname}: {ls}') + for dev in ls: + devname = dev.split('/')[-1].replace("_", "-") + crs.append({ + 'apiVersion': 'v1', + 'kind': 'PersistentVolume', + 'metadata': {'name': f'{remote.shortname}-{devname}'}, + 'spec': { + 'volumeMode': 'Block', + 'accessModes': ['ReadWriteOnce'], + 'capacity': {'storage': '100Gi'}, # doesn't matter? + 'persistentVolumeReclaimPolicy': 'Recycle', + 'storageClassName': 'scratch', + 'local': {'path': dev}, + 'nodeAffinity': { + 'required': { + 'nodeSelectorTerms': [ + { + 'matchExpressions': [ + { + 'key': 'kubernetes.io/hostname', + 'operator': 'In', + 'values': [remote.shortname] + } + ] + } + ] + } + } + } + }) + # overwriting first few MB is enough to make k8s happy + remote.run(args=[ + 'sudo', 'dd', 'if=/dev/zero', f'of={dev}', 'bs=1M', 'count=10' + ]) + crs.append({ + 'kind': 'StorageClass', + 'apiVersion': 'storage.k8s.io/v1', + 'metadata': {'name': 'scratch'}, + 'provisioner': 'kubernetes.io/no-provisioner', + 'volumeBindingMode': 'WaitForFirstConsumer', + }) + y = yaml.dump_all(crs) + log.info('Creating PVs + StorageClass') + log.debug(y) + _kubectl(ctx, config, ['create', '-f', '-'], stdin=y) + + yield + + +@contextlib.contextmanager +def final(ctx, config): + cluster_name = config['cluster'] + + # remove master node taint + _kubectl(ctx, config, [ + 'taint', 'node', + ctx.kubeadm[cluster_name].bootstrap_remote.shortname, + 'node-role.kubernetes.io/master-', + run.Raw('||'), + 'true', + ]) + + yield + + +@contextlib.contextmanager +def task(ctx, config): + if not config: + config = {} + assert isinstance(config, dict), \ + "task only supports a dictionary for configuration" + + log.info('Kubeadm start') + + overrides = ctx.config.get('overrides', {}) + teuthology.deep_merge(config, overrides.get('kubeadm', {})) + log.info('Config: ' + str(config)) + + # set up cluster context + if not hasattr(ctx, 'kubeadm'): + ctx.kubeadm = {} + if 'cluster' not in config: + config['cluster'] = 'kubeadm' + cluster_name = config['cluster'] + if cluster_name not in ctx.kubeadm: + ctx.kubeadm[cluster_name] = argparse.Namespace() + + with contextutil.nested( + lambda: preflight(ctx, config), + lambda: allocate_pod_subnet(ctx, config), + lambda: kubeadm_install(ctx, config), + lambda: kubeadm_init_join(ctx, config), + lambda: kubectl_config(ctx, config), + lambda: pod_network(ctx, config), + lambda: setup_pvs(ctx, config), + lambda: final(ctx, config), + ): + try: + log.info('Kubeadm complete, yielding') + yield + + finally: + log.info('Tearing down kubeadm') diff --git a/ceph/qa/tasks/mgr/dashboard/test_crush_rule.py b/ceph/qa/tasks/mgr/dashboard/test_crush_rule.py index 30ce834a5..1e37553b2 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_crush_rule.py +++ b/ceph/qa/tasks/mgr/dashboard/test_crush_rule.py @@ -82,5 +82,6 @@ class CrushRuleTest(DashboardTestCase): self.assertStatus(200) self.assertSchemaBody(JObj({ 'names': JList(str), - 'nodes': JList(JObj({}, allow_unknown=True)) + 'nodes': JList(JObj({}, allow_unknown=True)), + 'roots': JList(int) })) diff --git a/ceph/qa/tasks/mgr/dashboard/test_host.py b/ceph/qa/tasks/mgr/dashboard/test_host.py index 49bb33533..124fff8d1 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_host.py +++ b/ceph/qa/tasks/mgr/dashboard/test_host.py @@ -146,7 +146,7 @@ class HostControllerTest(DashboardTestCase): class HostControllerNoOrchestratorTest(DashboardTestCase): def test_host_create(self): - self._post('/api/host?hostname=foo') + self._post('/api/host?hostname=foo', {'status': ''}, version='0.1') self.assertStatus(503) self.assertError(code='orchestrator_status_unavailable', component='orchestrator') diff --git a/ceph/qa/tasks/mgr/dashboard/test_rgw.py b/ceph/qa/tasks/mgr/dashboard/test_rgw.py index 5029c12d3..f545c7483 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_rgw.py +++ b/ceph/qa/tasks/mgr/dashboard/test_rgw.py @@ -433,6 +433,16 @@ class RgwBucketTest(RgwTestCase): self.assertEqual(data['lock_retention_period_days'], 15) self.assertEqual(data['lock_retention_period_years'], 0) self.assertStatus(200) + + # Update: Disabling bucket versioning should fail if object locking enabled + self._put('/api/rgw/bucket/teuth-test-bucket', + params={ + 'bucket_id': data['id'], + 'uid': 'teuth-test-user', + 'versioning_state': 'Suspended' + }) + self.assertStatus(409) + # Delete self._delete('/api/rgw/bucket/teuth-test-bucket') self.assertStatus(204) diff --git a/ceph/qa/tasks/mgr/test_dashboard.py b/ceph/qa/tasks/mgr/test_dashboard.py index b30175f4f..c3459ec02 100644 --- a/ceph/qa/tasks/mgr/test_dashboard.py +++ b/ceph/qa/tasks/mgr/test_dashboard.py @@ -39,6 +39,12 @@ class TestDashboard(MgrTestCase): self.wait_until_true(_check_connection, timeout=30) def test_standby(self): + # skip this test if mgr_standby_modules=false + if self.mgr_cluster.mon_manager.raw_cluster_cmd( + "config", "get", "mgr", "mgr_standby_modules").strip() == "false": + log.info("Skipping test_standby since mgr_standby_modules=false") + return + original_active_id = self.mgr_cluster.get_active_id() original_uri = self._get_uri("dashboard") log.info("Originally running manager '{}' at {}".format( diff --git a/ceph/qa/tasks/mgr/test_progress.py b/ceph/qa/tasks/mgr/test_progress.py index 0e03c2b66..69e918753 100644 --- a/ceph/qa/tasks/mgr/test_progress.py +++ b/ceph/qa/tasks/mgr/test_progress.py @@ -14,7 +14,7 @@ class TestProgress(MgrTestCase): # How long we expect to wait at most between taking an OSD out # and seeing the progress event pop up. - EVENT_CREATION_PERIOD = 5 + EVENT_CREATION_PERIOD = 15 WRITE_PERIOD = 30 diff --git a/ceph/qa/tasks/rook-ceph.conf b/ceph/qa/tasks/rook-ceph.conf new file mode 100644 index 000000000..38ac11e41 --- /dev/null +++ b/ceph/qa/tasks/rook-ceph.conf @@ -0,0 +1,41 @@ +[global] + +log to file = true + +mon clock drift allowed = 1.000 + +# replicate across OSDs, not hosts +osd crush chooseleaf type = 0 + +# enable some debugging +auth debug = true +ms die on old message = true +ms die on bug = true +debug asserts on shutdown = true + + +[osd] +# debugging +osd debug shutdown = true +osd debug op order = true +osd debug verify stray on activate = true +osd debug pg log writeout = true +osd debug verify cached snaps = true +osd debug verify missing on start = true +osd debug misdirected ops = true +osd op queue = debug_random +osd op queue cut off = debug_random +osd shutdown pgref assert = true +bdev debug aio = true +osd sloppy crc = true + + +[mon] +# rotate auth tickets quickly to exercise renewal paths +auth mon ticket ttl = 660 # 11m +auth service ticket ttl = 240 # 4m + +# don't complain about global id reclaim +mon_warn_on_insecure_global_id_reclaim = false +mon_warn_on_insecure_global_id_reclaim_allowed = false + diff --git a/ceph/qa/tasks/rook.py b/ceph/qa/tasks/rook.py new file mode 100644 index 000000000..bdd9e58dc --- /dev/null +++ b/ceph/qa/tasks/rook.py @@ -0,0 +1,638 @@ +""" +Rook cluster task +""" +import argparse +import configobj +import contextlib +import json +import logging +import os +import yaml +from io import BytesIO + +from tarfile import ReadError +from tasks.ceph_manager import CephManager +from teuthology import misc as teuthology +from teuthology.config import config as teuth_config +from teuthology.contextutil import safe_while +from teuthology.orchestra import run +from teuthology import contextutil +from tasks.ceph import healthy +from tasks.cephadm import update_archive_setting + +log = logging.getLogger(__name__) + + +def _kubectl(ctx, config, args, **kwargs): + cluster_name = config.get('cluster', 'ceph') + return ctx.rook[cluster_name].remote.run( + args=['kubectl'] + args, + **kwargs + ) + + +def shell(ctx, config): + """ + Run command(s) inside the rook tools container. + + tasks: + - kubeadm: + - rook: + - rook.shell: + - ceph -s + + or + + tasks: + - kubeadm: + - rook: + - rook.shell: + commands: + - ceph -s + + """ + if isinstance(config, list): + config = {'commands': config} + for cmd in config.get('commands', []): + if isinstance(cmd, str): + _shell(ctx, config, cmd.split(' ')) + else: + _shell(ctx, config, cmd) + + +def _shell(ctx, config, args, **kwargs): + cluster_name = config.get('cluster', 'ceph') + return _kubectl( + ctx, config, + [ + '-n', 'rook-ceph', + 'exec', + ctx.rook[cluster_name].toolbox, '--' + ] + args, + **kwargs + ) + + +@contextlib.contextmanager +def rook_operator(ctx, config): + cluster_name = config['cluster'] + rook_branch = config.get('rook_branch', 'master') + rook_git_url = config.get('rook_git_url', 'https://github.com/rook/rook') + + log.info(f'Cloning {rook_git_url} branch {rook_branch}') + ctx.rook[cluster_name].remote.run( + args=[ + 'rm', '-rf', 'rook', + run.Raw('&&'), + 'git', + 'clone', + '--single-branch', + '--branch', rook_branch, + rook_git_url, + 'rook', + ] + ) + + # operator.yaml + operator_yaml = ctx.rook[cluster_name].remote.read_file( + 'rook/cluster/examples/kubernetes/ceph/operator.yaml' + ) + rook_image = config.get('rook_image') + if rook_image: + log.info(f'Patching operator to use image {rook_image}') + crs = list(yaml.load_all(operator_yaml, Loader=yaml.FullLoader)) + assert len(crs) == 2 + crs[1]['spec']['template']['spec']['containers'][0]['image'] = rook_image + operator_yaml = yaml.dump_all(crs) + ctx.rook[cluster_name].remote.write_file('operator.yaml', operator_yaml) + + op_job = None + try: + log.info('Deploying operator') + _kubectl(ctx, config, [ + 'create', + '-f', 'rook/cluster/examples/kubernetes/ceph/crds.yaml', + '-f', 'rook/cluster/examples/kubernetes/ceph/common.yaml', + '-f', 'operator.yaml', + ]) + + # on centos: + if teuthology.get_distro(ctx) == 'centos': + _kubectl(ctx, config, [ + '-n', 'rook-ceph', + 'set', 'env', 'deploy/rook-ceph-operator', + 'ROOK_HOSTPATH_REQUIRES_PRIVILEGED=true' + ]) + + # wait for operator + op_name = None + with safe_while(sleep=10, tries=90, action="wait for operator") as proceed: + while not op_name and proceed(): + p = _kubectl( + ctx, config, + ['-n', 'rook-ceph', 'get', 'pods', '-l', 'app=rook-ceph-operator'], + stdout=BytesIO(), + ) + for line in p.stdout.getvalue().decode('utf-8').strip().splitlines(): + name, ready, status, _ = line.split(None, 3) + if status == 'Running': + op_name = name + break + + # log operator output + op_job = _kubectl( + ctx, + config, + ['-n', 'rook-ceph', 'logs', '-f', op_name], + wait=False, + logger=log.getChild('operator'), + ) + + yield + + except Exception as e: + log.exception(e) + raise + + finally: + log.info('Cleaning up rook operator') + _kubectl(ctx, config, [ + 'delete', + '-f', 'operator.yaml', + ]) + if False: + # don't bother since we'll tear down k8s anyway (and this mysteriously + # fails sometimes when deleting some of the CRDs... not sure why!) + _kubectl(ctx, config, [ + 'delete', + '-f', 'rook/cluster/examples/kubernetes/ceph/common.yaml', + ]) + _kubectl(ctx, config, [ + 'delete', + '-f', 'rook/cluster/examples/kubernetes/ceph/crds.yaml', + ]) + ctx.rook[cluster_name].remote.run(args=['rm', '-rf', 'rook', 'operator.yaml']) + if op_job: + op_job.wait() + run.wait( + ctx.cluster.run( + args=[ + 'sudo', 'rm', '-rf', '/var/lib/rook' + ] + ) + ) + + +@contextlib.contextmanager +def ceph_log(ctx, config): + cluster_name = config['cluster'] + + log_dir = '/var/lib/rook/rook-ceph/log' + update_archive_setting(ctx, 'log', log_dir) + + try: + yield + + except Exception: + # we need to know this below + ctx.summary['success'] = False + raise + + finally: + log.info('Checking cluster log for badness...') + def first_in_ceph_log(pattern, excludes): + """ + Find the first occurrence of the pattern specified in the Ceph log, + Returns None if none found. + + :param pattern: Pattern scanned for. + :param excludes: Patterns to ignore. + :return: First line of text (or None if not found) + """ + args = [ + 'sudo', + 'egrep', pattern, + f'{log_dir}/ceph.log', + ] + if excludes: + for exclude in excludes: + args.extend([run.Raw('|'), 'egrep', '-v', exclude]) + args.extend([ + run.Raw('|'), 'head', '-n', '1', + ]) + r = ctx.rook[cluster_name].remote.run( + stdout=BytesIO(), + args=args, + ) + stdout = r.stdout.getvalue().decode() + if stdout: + return stdout + return None + + if first_in_ceph_log('\[ERR\]|\[WRN\]|\[SEC\]', + config.get('log-ignorelist')) is not None: + log.warning('Found errors (ERR|WRN|SEC) in cluster log') + ctx.summary['success'] = False + # use the most severe problem as the failure reason + if 'failure_reason' not in ctx.summary: + for pattern in ['\[SEC\]', '\[ERR\]', '\[WRN\]']: + match = first_in_ceph_log(pattern, config['log-ignorelist']) + if match is not None: + ctx.summary['failure_reason'] = \ + '"{match}" in cluster log'.format( + match=match.rstrip('\n'), + ) + break + + if ctx.archive is not None and \ + not (ctx.config.get('archive-on-error') and ctx.summary['success']): + # and logs + log.info('Compressing logs...') + run.wait( + ctx.cluster.run( + args=[ + 'sudo', + 'find', + log_dir, + '-name', + '*.log', + '-print0', + run.Raw('|'), + 'sudo', + 'xargs', + '-0', + '--no-run-if-empty', + '--', + 'gzip', + '--', + ], + wait=False, + ), + ) + + log.info('Archiving logs...') + path = os.path.join(ctx.archive, 'remote') + try: + os.makedirs(path) + except OSError: + pass + for remote in ctx.cluster.remotes.keys(): + sub = os.path.join(path, remote.name) + try: + os.makedirs(sub) + except OSError: + pass + try: + teuthology.pull_directory(remote, log_dir, + os.path.join(sub, 'log')) + except ReadError: + pass + + +def build_initial_config(ctx, config): + path = os.path.join(os.path.dirname(__file__), 'rook-ceph.conf') + conf = configobj.ConfigObj(path, file_error=True) + + # overrides + for section, keys in config.get('conf',{}).items(): + for key, value in keys.items(): + log.info(" override: [%s] %s = %s" % (section, key, value)) + if section not in conf: + conf[section] = {} + conf[section][key] = value + + return conf + + +@contextlib.contextmanager +def rook_cluster(ctx, config): + cluster_name = config['cluster'] + + # count how many OSDs we'll create + num_devs = 0 + num_hosts = 0 + for remote in ctx.cluster.remotes.keys(): + ls = remote.read_file('/scratch_devs').decode('utf-8').strip().splitlines() + num_devs += len(ls) + num_hosts += 1 + ctx.rook[cluster_name].num_osds = num_devs + + # config + config = build_initial_config(ctx, config) + config_fp = BytesIO() + config.write(config_fp) + log.info(f'Config:\n{config_fp.getvalue()}') + _kubectl(ctx, config, ['create', '-f', '-'], stdin=yaml.dump({ + 'apiVersion': 'v1', + 'kind': 'ConfigMap', + 'metadata': { + 'name': 'rook-config-override', + 'namespace': 'rook-ceph'}, + 'data': { + 'config': config_fp.getvalue() + } + })) + + # cluster + cluster = { + 'apiVersion': 'ceph.rook.io/v1', + 'kind': 'CephCluster', + 'metadata': {'name': 'rook-ceph', 'namespace': 'rook-ceph'}, + 'spec': { + 'cephVersion': { + 'image': ctx.rook[cluster_name].image, + 'allowUnsupported': True, + }, + 'dataDirHostPath': '/var/lib/rook', + 'skipUpgradeChecks': True, + 'mgr': { + 'count': 1, + 'modules': [ + { 'name': 'rook', 'enabled': True }, + ], + }, + 'mon': { + 'count': num_hosts, + 'allowMultiplePerNode': True, + }, + 'storage': { + 'storageClassDeviceSets': [ + { + 'name': 'scratch', + 'count': num_devs, + 'portable': False, + 'volumeClaimTemplates': [ + { + 'metadata': {'name': 'data'}, + 'spec': { + 'resources': { + 'requests': { + 'storage': '10Gi' # <= (lte) the actual PV size + } + }, + 'storageClassName': 'scratch', + 'volumeMode': 'Block', + 'accessModes': ['ReadWriteOnce'], + }, + }, + ], + } + ], + }, + } + } + teuthology.deep_merge(cluster['spec'], config.get('spec', {})) + + cluster_yaml = yaml.dump(cluster) + log.info(f'Cluster:\n{cluster_yaml}') + try: + ctx.rook[cluster_name].remote.write_file('cluster.yaml', cluster_yaml) + _kubectl(ctx, config, ['create', '-f', 'cluster.yaml']) + yield + + except Exception as e: + log.exception(e) + raise + + finally: + _kubectl(ctx, config, ['delete', '-f', 'cluster.yaml'], check_status=False) + + # wait for cluster to shut down + log.info('Waiting for cluster to stop') + running = True + with safe_while(sleep=5, tries=100, action="wait for teardown") as proceed: + while running and proceed(): + p = _kubectl( + ctx, config, + ['-n', 'rook-ceph', 'get', 'pods'], + stdout=BytesIO(), + ) + running = False + for line in p.stdout.getvalue().decode('utf-8').strip().splitlines(): + name, ready, status, _ = line.split(None, 3) + if ( + name != 'NAME' + and not name.startswith('csi-') + and not name.startswith('rook-ceph-operator-') + and not name.startswith('rook-ceph-tools-') + ): + running = True + break + + _kubectl( + ctx, config, + ['-n', 'rook-ceph', 'delete', 'configmap', 'rook-config-override'], + check_status=False, + ) + ctx.rook[cluster_name].remote.run(args=['rm', '-f', 'cluster.yaml']) + + +@contextlib.contextmanager +def rook_toolbox(ctx, config): + cluster_name = config['cluster'] + try: + _kubectl(ctx, config, [ + 'create', + '-f', 'rook/cluster/examples/kubernetes/ceph/toolbox.yaml', + ]) + + log.info('Waiting for tools container to start') + toolbox = None + with safe_while(sleep=5, tries=100, action="wait for toolbox") as proceed: + while not toolbox and proceed(): + p = _kubectl( + ctx, config, + ['-n', 'rook-ceph', 'get', 'pods', '-l', 'app=rook-ceph-tools'], + stdout=BytesIO(), + ) + for line in p.stdout.getvalue().decode('utf-8').strip().splitlines(): + name, ready, status, _ = line.split(None, 3) + if status == 'Running': + toolbox = name + break + ctx.rook[cluster_name].toolbox = toolbox + yield + + except Exception as e: + log.exception(e) + raise + + finally: + _kubectl(ctx, config, [ + 'delete', + '-f', 'rook/cluster/examples/kubernetes/ceph/toolbox.yaml', + ], check_status=False) + + +@contextlib.contextmanager +def wait_for_osds(ctx, config): + cluster_name = config.get('cluster', 'ceph') + + want = ctx.rook[cluster_name].num_osds + log.info(f'Waiting for {want} OSDs') + with safe_while(sleep=10, tries=90, action="check osd count") as proceed: + while proceed(): + p = _shell(ctx, config, ['ceph', 'osd', 'stat', '-f', 'json'], + stdout=BytesIO(), + check_status=False) + if p.exitstatus == 0: + r = json.loads(p.stdout.getvalue().decode('utf-8')) + have = r.get('num_up_osds', 0) + if have == want: + break + log.info(f' have {have}/{want} OSDs') + + yield + + +@contextlib.contextmanager +def ceph_config_keyring(ctx, config): + # get config and push to hosts + log.info('Distributing ceph config and client.admin keyring') + p = _shell(ctx, config, ['cat', '/etc/ceph/ceph.conf'], stdout=BytesIO()) + conf = p.stdout.getvalue() + p = _shell(ctx, config, ['cat', '/etc/ceph/keyring'], stdout=BytesIO()) + keyring = p.stdout.getvalue() + ctx.cluster.run(args=['sudo', 'mkdir', '-p', '/etc/ceph']) + for remote in ctx.cluster.remotes.keys(): + remote.write_file( + '/etc/ceph/ceph.conf', + conf, + sudo=True, + ) + remote.write_file( + '/etc/ceph/keyring', + keyring, + sudo=True, + ) + + try: + yield + + except Exception as e: + log.exception(e) + raise + + finally: + log.info('Cleaning up config and client.admin keyring') + ctx.cluster.run(args=[ + 'sudo', 'rm', '-f', + '/etc/ceph/ceph.conf', + '/etc/ceph/ceph.client.admin.keyring' + ]) + + +@contextlib.contextmanager +def ceph_clients(ctx, config): + cluster_name = config['cluster'] + + log.info('Setting up client nodes...') + clients = ctx.cluster.only(teuthology.is_type('client', cluster_name)) + for remote, roles_for_host in clients.remotes.items(): + for role in teuthology.cluster_roles_of_type(roles_for_host, 'client', + cluster_name): + name = teuthology.ceph_role(role) + client_keyring = '/etc/ceph/{0}.{1}.keyring'.format(cluster_name, + name) + r = _shell(ctx, config, + args=[ + 'ceph', 'auth', + 'get-or-create', name, + 'mon', 'allow *', + 'osd', 'allow *', + 'mds', 'allow *', + 'mgr', 'allow *', + ], + stdout=BytesIO(), + ) + keyring = r.stdout.getvalue() + remote.write_file(client_keyring, keyring, sudo=True, mode='0644') + yield + + +@contextlib.contextmanager +def task(ctx, config): + """ + Deploy rook-ceph cluster + + tasks: + - kubeadm: + - rook: + branch: wip-foo + spec: + mon: + count: 1 + + The spec item is deep-merged against the cluster.yaml. The branch, sha1, or + image items are used to determine the Ceph container image. + """ + if not config: + config = {} + assert isinstance(config, dict), \ + "task only supports a dictionary for configuration" + + log.info('Rook start') + + overrides = ctx.config.get('overrides', {}) + teuthology.deep_merge(config, overrides.get('ceph', {})) + teuthology.deep_merge(config, overrides.get('rook', {})) + log.info('Config: ' + str(config)) + + # set up cluster context + if not hasattr(ctx, 'rook'): + ctx.rook = {} + if 'cluster' not in config: + config['cluster'] = 'ceph' + cluster_name = config['cluster'] + if cluster_name not in ctx.rook: + ctx.rook[cluster_name] = argparse.Namespace() + + ctx.rook[cluster_name].remote = list(ctx.cluster.remotes.keys())[0] + + # image + teuth_defaults = teuth_config.get('defaults', {}) + cephadm_defaults = teuth_defaults.get('cephadm', {}) + containers_defaults = cephadm_defaults.get('containers', {}) + container_image_name = containers_defaults.get('image', None) + if 'image' in config: + ctx.rook[cluster_name].image = config.get('image') + else: + sha1 = config.get('sha1') + flavor = config.get('flavor', 'default') + if sha1: + if flavor == "crimson": + ctx.rook[cluster_name].image = container_image_name + ':' + sha1 + '-' + flavor + else: + ctx.rook[cluster_name].image = container_image_name + ':' + sha1 + else: + # hmm, fall back to branch? + branch = config.get('branch', 'master') + ctx.rook[cluster_name].image = container_image_name + ':' + branch + log.info('Ceph image is %s' % ctx.rook[cluster_name].image) + + with contextutil.nested( + lambda: rook_operator(ctx, config), + lambda: ceph_log(ctx, config), + lambda: rook_cluster(ctx, config), + lambda: rook_toolbox(ctx, config), + lambda: wait_for_osds(ctx, config), + lambda: ceph_config_keyring(ctx, config), + lambda: ceph_clients(ctx, config), + ): + if not hasattr(ctx, 'managers'): + ctx.managers = {} + ctx.managers[cluster_name] = CephManager( + ctx.rook[cluster_name].remote, + ctx=ctx, + logger=log.getChild('ceph_manager.' + cluster_name), + cluster=cluster_name, + rook=True, + ) + try: + if config.get('wait-for-healthy', True): + healthy(ctx=ctx, config=config) + log.info('Rook complete, yielding') + yield + + finally: + log.info('Tearing down rook') diff --git a/ceph/qa/tasks/userdata_setup.yaml b/ceph/qa/tasks/userdata_setup.yaml index 7271925c9..9aa2d0396 100644 --- a/ceph/qa/tasks/userdata_setup.yaml +++ b/ceph/qa/tasks/userdata_setup.yaml @@ -14,6 +14,8 @@ #!/usr/bin/env bash # mount a NFS share for storing logs + sed -i 's/archive.ubuntu.com/old-releases.ubuntu.com/' /etc/apt/sources.list + sed -i 's/security.ubuntu.com/old-releases.ubuntu.com/' /etc/apt/sources.list apt-get update apt-get -y install nfs-common mkdir /mnt/log diff --git a/ceph/qa/tasks/vip.py b/ceph/qa/tasks/vip.py index d6d794ef0..d4d27b85e 100644 --- a/ceph/qa/tasks/vip.py +++ b/ceph/qa/tasks/vip.py @@ -3,6 +3,7 @@ import ipaddress import logging import re +from teuthology import misc as teuthology from teuthology.config import config as teuth_config log = logging.getLogger(__name__) @@ -34,6 +35,38 @@ def echo(ctx, config): log.info(subst_vip(ctx, config)) +def exec(ctx, config): + """ + This is similar to the standard 'exec' task, but does the VIP substitutions. + """ + assert isinstance(config, dict), "task exec got invalid config" + + testdir = teuthology.get_testdir(ctx) + + if 'all-roles' in config and len(config) == 1: + a = config['all-roles'] + roles = teuthology.all_roles(ctx.cluster) + config = dict((id_, a) for id_ in roles if not id_.startswith('host.')) + elif 'all-hosts' in config and len(config) == 1: + a = config['all-hosts'] + roles = teuthology.all_roles(ctx.cluster) + config = dict((id_, a) for id_ in roles if id_.startswith('host.')) + + for role, ls in config.items(): + (remote,) = ctx.cluster.only(role).remotes.keys() + log.info('Running commands on role %s host %s', role, remote.name) + for c in ls: + c.replace('$TESTDIR', testdir) + remote.run( + args=[ + 'sudo', + 'TESTDIR={tdir}'.format(tdir=testdir), + 'bash', + '-c', + subst_vip(ctx, c)], + ) + + def map_vips(mip, count): for mapping in teuth_config.get('vip', []): mnet = ipaddress.ip_network(mapping['machine_subnet']) diff --git a/ceph/qa/tasks/workunit.py b/ceph/qa/tasks/workunit.py index 491f68308..371d2a2dd 100644 --- a/ceph/qa/tasks/workunit.py +++ b/ceph/qa/tasks/workunit.py @@ -400,6 +400,7 @@ def _run_tests(ctx, refspec, role, tests, env, basedir, run.Raw('PATH=$PATH:/usr/sbin'), run.Raw('CEPH_BASE={dir}'.format(dir=clonedir)), run.Raw('CEPH_ROOT={dir}'.format(dir=clonedir)), + run.Raw('CEPH_MNT={dir}'.format(dir=mnt)), ] if env is not None: for var, val in env.items(): diff --git a/ceph/qa/workunits/cephadm/test_cephadm.sh b/ceph/qa/workunits/cephadm/test_cephadm.sh index 8f5144c2c..2bc94b88c 100755 --- a/ceph/qa/workunits/cephadm/test_cephadm.sh +++ b/ceph/qa/workunits/cephadm/test_cephadm.sh @@ -415,8 +415,11 @@ expect_false $CEPHADM rm-daemon --fsid $FSID --name mon.a # mgr does not $CEPHADM rm-daemon --fsid $FSID --name mgr.x +expect_false $CEPHADM zap-osds --fsid $FSID +$CEPHADM zap-osds --fsid $FSID --force + ## rm-cluster -expect_false $CEPHADM rm-cluster --fsid $FSID -$CEPHADM rm-cluster --fsid $FSID --force +expect_false $CEPHADM rm-cluster --fsid $FSID --zap-osds +$CEPHADM rm-cluster --fsid $FSID --force --zap-osds echo PASS diff --git a/ceph/qa/workunits/cephadm/test_dashboard_e2e.sh b/ceph/qa/workunits/cephadm/test_dashboard_e2e.sh index 5764ecb5d..bd37154d9 100755 --- a/ceph/qa/workunits/cephadm/test_dashboard_e2e.sh +++ b/ceph/qa/workunits/cephadm/test_dashboard_e2e.sh @@ -74,10 +74,12 @@ ceph device monitoring off ceph tell mon.\* injectargs '--mon-allow-pool-delete=true' ceph osd pool rm device_health_metrics device_health_metrics --yes-i-really-really-mean-it -# Take `orch device ls` as ground truth. +# Take `orch device ls` and `orch ps` as ground truth. ceph orch device ls --refresh +ceph orch ps --refresh sleep 10 # the previous call is asynchronous ceph orch device ls --format=json | tee cypress/fixtures/orchestrator/inventory.json +ceph orch ps --format=json | tee cypress/fixtures/orchestrator/services.json DASHBOARD_ADMIN_SECRET_FILE="/tmp/dashboard-admin-secret.txt" printf 'admin' > "${DASHBOARD_ADMIN_SECRET_FILE}" @@ -91,10 +93,23 @@ find cypress # List all specs cypress_run "orchestrator/01-hosts.e2e-spec.ts" +ceph orch apply rgw foo --placement=3 +sleep 15 +ceph orch device ls --refresh +ceph orch ps --refresh +sleep 10 # the previous call is asynchronous +ceph orch device ls --format=json | tee cypress/fixtures/orchestrator/inventory.json +ceph orch ps --format=json | tee cypress/fixtures/orchestrator/services.json + +cypress_run "orchestrator/01-hosts-force-maintenance.e2e-spec.ts" + # Hosts are removed and added in the previous step. Do a refresh again. +ceph orch rm rgw.foo ceph orch device ls --refresh +ceph orch ps --refresh sleep 10 ceph orch device ls --format=json | tee cypress/fixtures/orchestrator/inventory.json +ceph orch ps --format=json | tee cypress/fixtures/orchestrator/services.json cypress_run "orchestrator/02-hosts-inventory.e2e-spec.ts" cypress_run "orchestrator/03-inventory.e2e-spec.ts" diff --git a/ceph/qa/workunits/fs/full/subvolume_rm.sh b/ceph/qa/workunits/fs/full/subvolume_rm.sh new file mode 100755 index 000000000..d0f9e2403 --- /dev/null +++ b/ceph/qa/workunits/fs/full/subvolume_rm.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -ex + +# This testcase tests the scenario of the 'ceph fs subvolume rm' mgr command +# when the osd is full. The command used to hang. The osd is of the size 1GB. +# The subvolume is created and 500MB file is written. The full-ratios are +# set below 500MB such that the osd is treated as full. Now the subvolume is +# is removed. This should be successful with the introduction of FULL +# capabilities which the mgr holds. + +set -e +expect_failure() { + if "$@"; then return 1; else return 0; fi +} + +ceph fs subvolume create cephfs sub_0 +subvol_path=$(ceph fs subvolume getpath cephfs sub_0 2>/dev/null) + +#For debugging +echo "Before write" +df -h +ceph osd df + +sudo dd if=/dev/urandom of=$CEPH_MNT$subvol_path/500MB_file-1 status=progress bs=1M count=500 + +ceph osd set-full-ratio 0.2 +ceph osd set-nearfull-ratio 0.16 +ceph osd set-backfillfull-ratio 0.18 + +timeout=30 +while [ $timeout -gt 0 ] +do + health=$(ceph health detail) + [[ $health = *"OSD_FULL"* ]] && echo "OSD is full" && break + echo "Wating for osd to be full: $timeout" + sleep 1 + let "timeout-=1" +done + +#For debugging +echo "After ratio set" +df -h +ceph osd df + +#Delete subvolume +ceph fs subvolume rm cephfs sub_0 + +#Validate subvolume is deleted +expect_failure ceph fs subvolume info cephfs sub_0 + +#Wait for subvolume to delete data +trashdir=$CEPH_MNT/volumes/_deleting +timeout=30 +while [ $timeout -gt 0 ] +do + [ -z "$(sudo ls -A $trashdir)" ] && echo "Trash directory $trashdir is empty" && break + echo "Wating for trash dir to be empty: $timeout" + sleep 1 + let "timeout-=1" +done + +echo OK diff --git a/ceph/qa/workunits/rbd/cli_generic.sh b/ceph/qa/workunits/rbd/cli_generic.sh index e363109c2..e6a7cb759 100755 --- a/ceph/qa/workunits/rbd/cli_generic.sh +++ b/ceph/qa/workunits/rbd/cli_generic.sh @@ -485,21 +485,219 @@ test_purge() { echo "testing trash purge..." remove_images + rbd trash ls | wc -l | grep 0 + rbd trash purge + + rbd create $RBD_CREATE_ARGS --size 256 testimg1 + rbd create $RBD_CREATE_ARGS --size 256 testimg2 + rbd trash mv testimg1 + rbd trash mv testimg2 + rbd trash ls | wc -l | grep 2 + rbd trash purge + rbd trash ls | wc -l | grep 0 + + rbd create $RBD_CREATE_ARGS --size 256 testimg1 + rbd create $RBD_CREATE_ARGS --size 256 testimg2 + rbd trash mv testimg1 --expires-at "1 hour" + rbd trash mv testimg2 --expires-at "3 hours" + rbd trash ls | wc -l | grep 2 + rbd trash purge + rbd trash ls | wc -l | grep 2 + rbd trash purge --expired-before "now + 2 hours" + rbd trash ls | wc -l | grep 1 + rbd trash ls | grep testimg2 + rbd trash purge --expired-before "now + 4 hours" + rbd trash ls | wc -l | grep 0 + + rbd create $RBD_CREATE_ARGS --size 256 testimg1 + rbd snap create testimg1@snap # pin testimg1 + rbd create $RBD_CREATE_ARGS --size 256 testimg2 + rbd create $RBD_CREATE_ARGS --size 256 testimg3 + rbd trash mv testimg1 + rbd trash mv testimg2 + rbd trash mv testimg3 + rbd trash ls | wc -l | grep 3 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 1 + rbd trash ls | grep testimg1 + ID=$(rbd trash ls | awk '{ print $1 }') + rbd snap purge --image-id $ID + rbd trash purge + rbd trash ls | wc -l | grep 0 + + rbd create $RBD_CREATE_ARGS --size 256 testimg1 + rbd create $RBD_CREATE_ARGS --size 256 testimg2 + rbd snap create testimg2@snap # pin testimg2 + rbd create $RBD_CREATE_ARGS --size 256 testimg3 + rbd trash mv testimg1 + rbd trash mv testimg2 + rbd trash mv testimg3 + rbd trash ls | wc -l | grep 3 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 1 + rbd trash ls | grep testimg2 + ID=$(rbd trash ls | awk '{ print $1 }') + rbd snap purge --image-id $ID rbd trash purge rbd trash ls | wc -l | grep 0 - rbd create $RBD_CREATE_ARGS foo -s 1 - rbd create $RBD_CREATE_ARGS bar -s 1 + rbd create $RBD_CREATE_ARGS --size 256 testimg1 + rbd create $RBD_CREATE_ARGS --size 256 testimg2 + rbd create $RBD_CREATE_ARGS --size 256 testimg3 + rbd snap create testimg3@snap # pin testimg3 + rbd trash mv testimg1 + rbd trash mv testimg2 + rbd trash mv testimg3 + rbd trash ls | wc -l | grep 3 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 1 + rbd trash ls | grep testimg3 + ID=$(rbd trash ls | awk '{ print $1 }') + rbd snap purge --image-id $ID + rbd trash purge + rbd trash ls | wc -l | grep 0 - rbd trash mv foo --expires-at "10 sec" - rbd trash mv bar --expires-at "30 sec" + # test purging a clone with a chain of parents + rbd create $RBD_CREATE_ARGS --size 256 testimg1 + rbd snap create testimg1@snap + rbd clone --rbd-default-clone-format=2 testimg1@snap testimg2 + rbd snap rm testimg1@snap + rbd create $RBD_CREATE_ARGS --size 256 testimg3 + rbd snap create testimg2@snap + rbd clone --rbd-default-clone-format=2 testimg2@snap testimg4 + rbd clone --rbd-default-clone-format=2 testimg2@snap testimg5 + rbd snap rm testimg2@snap + rbd snap create testimg4@snap + rbd clone --rbd-default-clone-format=2 testimg4@snap testimg6 + rbd snap rm testimg4@snap + rbd trash mv testimg1 + rbd trash mv testimg2 + rbd trash mv testimg3 + rbd trash mv testimg4 + rbd trash ls | wc -l | grep 4 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 3 + rbd trash ls | grep testimg1 + rbd trash ls | grep testimg2 + rbd trash ls | grep testimg4 + rbd trash mv testimg6 + rbd trash ls | wc -l | grep 4 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 2 + rbd trash ls | grep testimg1 + rbd trash ls | grep testimg2 + rbd trash mv testimg5 + rbd trash ls | wc -l | grep 3 + rbd trash purge + rbd trash ls | wc -l | grep 0 - rbd trash purge --expired-before "now + 10 sec" - rbd trash ls | grep -v foo | wc -l | grep 1 - rbd trash ls | grep bar + rbd create $RBD_CREATE_ARGS --size 256 testimg1 + rbd snap create testimg1@snap + rbd clone --rbd-default-clone-format=2 testimg1@snap testimg2 + rbd snap rm testimg1@snap + rbd create $RBD_CREATE_ARGS --size 256 testimg3 + rbd snap create testimg3@snap # pin testimg3 + rbd snap create testimg2@snap + rbd clone --rbd-default-clone-format=2 testimg2@snap testimg4 + rbd clone --rbd-default-clone-format=2 testimg2@snap testimg5 + rbd snap rm testimg2@snap + rbd snap create testimg4@snap + rbd clone --rbd-default-clone-format=2 testimg4@snap testimg6 + rbd snap rm testimg4@snap + rbd trash mv testimg1 + rbd trash mv testimg2 + rbd trash mv testimg3 + rbd trash mv testimg4 + rbd trash ls | wc -l | grep 4 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 4 + rbd trash mv testimg6 + rbd trash ls | wc -l | grep 5 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 3 + rbd trash ls | grep testimg1 + rbd trash ls | grep testimg2 + rbd trash ls | grep testimg3 + rbd trash mv testimg5 + rbd trash ls | wc -l | grep 4 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 1 + rbd trash ls | grep testimg3 + ID=$(rbd trash ls | awk '{ print $1 }') + rbd snap purge --image-id $ID + rbd trash purge + rbd trash ls | wc -l | grep 0 + + # test purging a clone with a chain of auto-delete parents + rbd create $RBD_CREATE_ARGS --size 256 testimg1 + rbd snap create testimg1@snap + rbd clone --rbd-default-clone-format=2 testimg1@snap testimg2 + rbd snap rm testimg1@snap + rbd create $RBD_CREATE_ARGS --size 256 testimg3 + rbd snap create testimg2@snap + rbd clone --rbd-default-clone-format=2 testimg2@snap testimg4 + rbd clone --rbd-default-clone-format=2 testimg2@snap testimg5 + rbd snap rm testimg2@snap + rbd snap create testimg4@snap + rbd clone --rbd-default-clone-format=2 testimg4@snap testimg6 + rbd snap rm testimg4@snap + rbd rm --rbd_move_parent_to_trash_on_remove=true testimg1 + rbd rm --rbd_move_parent_to_trash_on_remove=true testimg2 + rbd trash mv testimg3 + rbd rm --rbd_move_parent_to_trash_on_remove=true testimg4 + rbd trash ls | wc -l | grep 4 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 3 + rbd trash ls | grep testimg1 + rbd trash ls | grep testimg2 + rbd trash ls | grep testimg4 + rbd trash mv testimg6 + rbd trash ls | wc -l | grep 4 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 2 + rbd trash ls | grep testimg1 + rbd trash ls | grep testimg2 + rbd trash mv testimg5 + rbd trash ls | wc -l | grep 3 + rbd trash purge + rbd trash ls | wc -l | grep 0 - LAST_IMG=$(rbd trash ls | grep bar | awk '{print $1;}') - rbd trash rm $LAST_IMG --force --no-progress | grep -v '.' | wc -l | grep 0 + rbd create $RBD_CREATE_ARGS --size 256 testimg1 + rbd snap create testimg1@snap + rbd clone --rbd-default-clone-format=2 testimg1@snap testimg2 + rbd snap rm testimg1@snap + rbd create $RBD_CREATE_ARGS --size 256 testimg3 + rbd snap create testimg3@snap # pin testimg3 + rbd snap create testimg2@snap + rbd clone --rbd-default-clone-format=2 testimg2@snap testimg4 + rbd clone --rbd-default-clone-format=2 testimg2@snap testimg5 + rbd snap rm testimg2@snap + rbd snap create testimg4@snap + rbd clone --rbd-default-clone-format=2 testimg4@snap testimg6 + rbd snap rm testimg4@snap + rbd rm --rbd_move_parent_to_trash_on_remove=true testimg1 + rbd rm --rbd_move_parent_to_trash_on_remove=true testimg2 + rbd trash mv testimg3 + rbd rm --rbd_move_parent_to_trash_on_remove=true testimg4 + rbd trash ls | wc -l | grep 4 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 4 + rbd trash mv testimg6 + rbd trash ls | wc -l | grep 5 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 3 + rbd trash ls | grep testimg1 + rbd trash ls | grep testimg2 + rbd trash ls | grep testimg3 + rbd trash mv testimg5 + rbd trash ls | wc -l | grep 4 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 1 + rbd trash ls | grep testimg3 + ID=$(rbd trash ls | awk '{ print $1 }') + rbd snap purge --image-id $ID + rbd trash purge + rbd trash ls | wc -l | grep 0 } test_deep_copy_clone() { diff --git a/ceph/qa/workunits/rbd/qemu-iotests.sh b/ceph/qa/workunits/rbd/qemu-iotests.sh index ddb63dae3..1f13da9fc 100755 --- a/ceph/qa/workunits/rbd/qemu-iotests.sh +++ b/ceph/qa/workunits/rbd/qemu-iotests.sh @@ -5,11 +5,11 @@ # require the admin ceph user, as there's no way to pass the ceph user # to qemu-iotests currently. -testlist='001 002 003 004 005 008 009 010 011 021 025 032 033 055' +testlist='001 002 003 004 005 008 009 010 011 021 025 032 033' git clone https://github.com/qemu/qemu.git cd qemu -if lsb_release -da 2>&1 | grep -iq 'bionic'; then +if lsb_release -da 2>&1 | grep -iqE '(bionic|focal)'; then # Bionic requires a matching test harness git checkout v2.11.0 elif lsb_release -da 2>&1 | grep -iqE '(xenial|linux release 8)'; then @@ -34,11 +34,6 @@ then ln -s /usr/bin/qemu-nbd else QEMU='/usr/libexec/qemu-kvm' - - # disable test 055 since qemu-kvm (RHEL/CentOS) doesn't support the - # required QMP commands under EL7 and Python 3 is not supported by - # the test under EL8 - testlist=$(echo ${testlist} | sed "s/ 055//g") fi ln -s $QEMU bin/qemu diff --git a/ceph/qa/workunits/rgw/test_rgw_orphan_list.sh b/ceph/qa/workunits/rgw/test_rgw_orphan_list.sh index 4299078a1..34d550cea 100755 --- a/ceph/qa/workunits/rgw/test_rgw_orphan_list.sh +++ b/ceph/qa/workunits/rgw/test_rgw_orphan_list.sh @@ -1,12 +1,13 @@ #!/usr/bin/env bash -set -ex +# set -x +set -e # if defined, debug messages will be displayed and prepended with the string # debug="DEBUG" -huge_size=2222 # in megabytes -big_size=6 # in megabytes +huge_size=5100 # in megabytes +big_size=7 # in megabytes huge_obj=/tmp/huge_obj.temp.$$ big_obj=/tmp/big_obj.temp.$$ @@ -160,7 +161,6 @@ mys3uploadkill() { exit 1 fi - set -v local_file="$1" remote_bkt="$2" remote_obj="$3" @@ -229,8 +229,16 @@ mys3cmd ls s3://multipart-bkt bkt="incomplete-mp-bkt-1" mys3cmd mb s3://$bkt -mys3uploadkill $huge_obj $bkt incomplete-mp-obj-1 $fifo 20 -mys3uploadkill $huge_obj $bkt incomplete-mp-obj-2 $fifo 100 + +mys3uploadkill $huge_obj $bkt incomplete-mp-obj-c $fifo 20 + +# generate an incomplete multipart with more than 1,000 parts +mys3uploadkill $huge_obj $bkt incomplete-mp-obj-b $fifo 1005 + +# generate more than 1000 incomplet multiparts +for c in $(seq 1005) ;do + mys3uploadkill $huge_obj $bkt incomplete-mp-obj-c-$c $fifo 3 +done #################################### # resharded bucket diff --git a/ceph/src/.git_version b/ceph/src/.git_version index cc4d4a67e..278f3aee5 100644 --- a/ceph/src/.git_version +++ b/ceph/src/.git_version @@ -1,2 +1,2 @@ -3cbe25cde3cfa028984618ad32de9edc4c1eaed0 -16.2.4 +0883bdea7337b95e4b611c768c0279868462204a +16.2.5 diff --git a/ceph/src/CMakeLists.txt b/ceph/src/CMakeLists.txt index 2a8056615..011408ceb 100644 --- a/ceph/src/CMakeLists.txt +++ b/ceph/src/CMakeLists.txt @@ -808,6 +808,13 @@ if(WITH_LIBCEPHSQLITE) set(cephsqlite_srcs libcephsqlite.cc SimpleRADOSStriper.cc) add_library(cephsqlite ${CEPH_SHARED} ${cephsqlite_srcs}) target_link_libraries(cephsqlite PRIVATE cls_lock_client librados ceph-common SQLite3::SQLite3 ${EXTRALIBS}) + set_target_properties(cephsqlite PROPERTIES + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON) + if(NOT APPLE AND NOT WIN32) + set_property(TARGET cephsqlite APPEND_STRING PROPERTY + LINK_FLAGS " -Wl,--exclude-libs,ALL") + endif() install(TARGETS cephsqlite DESTINATION ${CMAKE_INSTALL_LIBDIR}) endif(WITH_LIBCEPHSQLITE) diff --git a/ceph/src/SimpleRADOSStriper.h b/ceph/src/SimpleRADOSStriper.h index 2e7984870..c5d3f5090 100644 --- a/ceph/src/SimpleRADOSStriper.h +++ b/ceph/src/SimpleRADOSStriper.h @@ -27,7 +27,7 @@ #include "common/ceph_time.h" #include "common/perf_counters.h" -class SimpleRADOSStriper +class [[gnu::visibility("default")]] SimpleRADOSStriper { public: using aiocompletionptr = std::unique_ptr; diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/batch.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/batch.py index 40c0fea4e..114730ade 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/batch.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/batch.py @@ -106,7 +106,7 @@ def get_physical_fast_allocs(devices, type_, fast_slots_per_device, new_osds, ar requested_slots = fast_slots_per_device requested_size = getattr(args, '{}_size'.format(type_), 0) - if requested_size == 0: + if not requested_size or requested_size == 0: # no size argument was specified, check ceph.conf get_size_fct = getattr(prepare, 'get_{}_size'.format(type_)) requested_size = get_size_fct(lv_format=False) @@ -126,6 +126,7 @@ def get_physical_fast_allocs(devices, type_, fast_slots_per_device, new_osds, ar if requested_size: if requested_size <= abs_size: abs_size = requested_size + relative_size = int(abs_size) / dev_size else: mlogger.error( '{} was requested for {}, but only {} can be fulfilled'.format( diff --git a/ceph/src/ceph_mon.cc b/ceph/src/ceph_mon.cc index 71c53d6db..ed949bda0 100644 --- a/ceph/src/ceph_mon.cc +++ b/ceph/src/ceph_mon.cc @@ -123,6 +123,14 @@ int obtain_monmap(MonitorDBStore &store, bufferlist &bl) } } + if (store.exists("mon_sync", "temp_newer_monmap")) { + dout(10) << __func__ << " found temp_newer_monmap" << dendl; + int err = store.get("mon_sync", "temp_newer_monmap", bl); + ceph_assert(err == 0); + ceph_assert(bl.length() > 0); + return 0; + } + if (store.exists("mkfs", "monmap")) { dout(10) << __func__ << " found mkfs monmap" << dendl; int err = store.get("mkfs", "monmap", bl); @@ -210,6 +218,8 @@ static void usage() << " extract the monmap from the local monitor store and exit\n" << " --mon-data \n" << " where the mon store and keyring are located\n" + << " --set-crush-location =" + << " sets monitor's crush bucket location (only for stretch mode)" << std::endl; generic_server_usage(); } @@ -248,7 +258,7 @@ int main(int argc, const char **argv) bool compact = false; bool force_sync = false; bool yes_really = false; - std::string osdmapfn, inject_monmap, extract_monmap; + std::string osdmapfn, inject_monmap, extract_monmap, crush_loc; vector args; argv_to_vec(argc, argv, args); @@ -331,6 +341,8 @@ int main(int argc, const char **argv) inject_monmap = val; } else if (ceph_argparse_witharg(args, i, &val, "--extract-monmap", (char*)NULL)) { extract_monmap = val; + } else if (ceph_argparse_witharg(args, i, &val, "--set-crush-location", (char*)NULL)) { + crush_loc = val; } else { ++i; } @@ -896,6 +908,7 @@ int main(int argc, const char **argv) msgr->start(); mgr_msgr->start(); + mon->set_mon_crush_location(crush_loc); mon->init(); register_async_signal_handler_oneshot(SIGINT, handle_mon_signal); diff --git a/ceph/src/cephadm/cephadm b/ceph/src/cephadm/cephadm index fef1ac1fa..92bb8d617 100755 --- a/ceph/src/cephadm/cephadm +++ b/ceph/src/cephadm/cephadm @@ -61,6 +61,7 @@ DATA_DIR = '/var/lib/ceph' LOG_DIR = '/var/log/ceph' LOCK_DIR = '/run/cephadm' LOGROTATE_DIR = '/etc/logrotate.d' +SYSCTL_DIR = '/usr/lib/sysctl.d' UNIT_DIR = '/etc/systemd/system' LOG_DIR_MODE = 0o770 DATA_DIR_MODE = 0o700 @@ -109,6 +110,7 @@ class BaseConfig: self.data_dir: str = DATA_DIR self.log_dir: str = LOG_DIR self.logrotate_dir: str = LOGROTATE_DIR + self.sysctl_dir: str = SYSCTL_DIR self.unit_dir: str = UNIT_DIR self.verbose: bool = False self.timeout: Optional[int] = DEFAULT_TIMEOUT @@ -210,11 +212,9 @@ logging_config = { }, 'log_file': { 'level': 'DEBUG', - 'class': 'logging.handlers.RotatingFileHandler', + 'class': 'logging.handlers.WatchedFileHandler', 'formatter': 'cephadm', 'filename': '%s/cephadm.log' % LOG_DIR, - 'maxBytes': 1024000, - 'backupCount': 1, } }, 'loggers': { @@ -249,6 +249,18 @@ class Ceph(object): ################################## +class OSD(object): + @staticmethod + def get_sysctl_settings() -> List[str]: + return [ + '# allow a large number of OSDs', + 'fs.aio-max-nr = 1048576', + 'kernel.pid_max = 4194304', + ] + +################################## + + class Monitoring(object): """Define the configs for the monitoring containers""" @@ -267,7 +279,6 @@ class Monitoring(object): 'args': [ '--config.file=/etc/prometheus/prometheus.yml', '--storage.tsdb.path=/prometheus', - '--web.listen-address=:{}'.format(port_map['prometheus'][0]), ], 'config-json-files': [ 'prometheus.yml', @@ -298,7 +309,6 @@ class Monitoring(object): 'cpus': '2', 'memory': '2GB', 'args': [ - '--web.listen-address=:{}'.format(port_map['alertmanager'][0]), '--cluster.listen-address=:{}'.format(port_map['alertmanager'][1]), ], 'config-json-files': [ @@ -349,7 +359,7 @@ def populate_files(config_dir, config_files, uid, gid): config_file = os.path.join(config_dir, fname) config_content = dict_get_join(config_files, fname) logger.info('Write file: %s' % (config_file)) - with open(config_file, 'w') as f: + with open(config_file, 'w', encoding='utf-8') as f: os.fchown(f.fileno(), uid, gid) os.fchmod(f.fileno(), 0o600) f.write(config_content) @@ -491,36 +501,6 @@ class NFSGanesha(object): os.fchown(f.fileno(), uid, gid) f.write(self.rgw.get('keyring', '')) - def get_rados_grace_container(self, action): - # type: (str) -> CephContainer - """Container for a ganesha action on the grace db""" - entrypoint = '/usr/bin/ganesha-rados-grace' - - assert self.pool - args = ['--pool', self.pool] - if self.namespace: - args += ['--ns', self.namespace] - if self.userid: - args += ['--userid', self.userid] - args += [action, self.get_daemon_name()] - - data_dir = get_data_dir(self.fsid, self.ctx.data_dir, - self.daemon_type, self.daemon_id) - volume_mounts = self.get_container_mounts(data_dir) - envs = self.get_container_envs() - - logger.info('Creating RADOS grace for action: %s' % action) - c = CephContainer( - self.ctx, - image=self.image, - entrypoint=entrypoint, - args=args, - volume_mounts=volume_mounts, - cname=self.get_container_name(desc='grace-%s' % action), - envs=envs - ) - return c - ################################## @@ -732,6 +712,13 @@ class HAproxy(object): mounts[os.path.join(data_dir, 'haproxy')] = '/var/lib/haproxy' return mounts + @staticmethod + def get_sysctl_settings() -> List[str]: + return [ + '# IP forwarding', + 'net.ipv4.ip_forward = 1', + ] + ################################## @@ -811,12 +798,12 @@ class Keepalived(object): return envs @staticmethod - def get_prestart(): - return ( - '# keepalived needs IP forwarding and non-local bind\n' - 'sysctl net.ipv4.ip_forward=1\n' - 'sysctl net.ipv4.ip_nonlocal_bind=1\n' - ) + def get_sysctl_settings() -> List[str]: + return [ + '# IP forwarding and non-local bind', + 'net.ipv4.ip_forward = 1', + 'net.ipv4.ip_nonlocal_bind = 1', + ] def extract_uid_gid_keepalived(self): # better directory for this? @@ -1020,13 +1007,15 @@ def attempt_bind(ctx, s, address, port): try: s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind((address, port)) - except (socket.error, OSError) as e: # py2 and py3 + except OSError as e: if e.errno == errno.EADDRINUSE: msg = 'Cannot bind to IP %s port %d: %s' % (address, port, e) logger.warning(msg) raise PortOccupiedError(msg) else: - raise e + raise Error(e) + except Exception as e: + raise Error(e) finally: s.close() @@ -1190,7 +1179,7 @@ class FileLock(object): lock_id, lock_filename, poll_intervall ) time.sleep(poll_intervall) - except: # noqa + except Exception: # Something did go wrong, so decrement the counter. self._lock_counter = max(0, self._lock_counter - 1) @@ -1484,6 +1473,16 @@ def call_timeout(ctx, command, timeout): ################################## +def json_loads_retry(cli_func): + for sleep_secs in [1, 4, 4]: + try: + return json.loads(cli_func()) + except json.JSONDecodeError: + logger.debug('Invalid JSON. Retrying in %s seconds...' % sleep_secs) + time.sleep(sleep_secs) + return json.loads(cli_func()) + + def is_available(ctx, what, func): # type: (CephadmContext, str, Callable[[], bool]) -> None """ @@ -2061,10 +2060,11 @@ def check_units(ctx, units, enabler=None): def is_container_running(ctx: CephadmContext, name: str) -> bool: - out, err, ret = call_throws(ctx, [ - ctx.container_engine.path, 'ps', - '--format', '{{.Names}}']) - return name in out + out, err, ret = call(ctx, [ + ctx.container_engine.path, 'container', 'inspect', + '--format', '{{.State.Status}}', name + ]) + return out == 'running' def get_legacy_config_fsid(cluster, legacy_dir=None): @@ -2121,6 +2121,17 @@ def get_daemon_args(ctx, fsid, daemon_type, daemon_id): elif daemon_type in Monitoring.components: metadata = Monitoring.components[daemon_type] r += metadata.get('args', list()) + # set ip and port to bind to for nodeexporter,alertmanager,prometheus + if daemon_type != 'grafana': + ip = '' + port = Monitoring.port_map[daemon_type][0] + if 'meta_json' in ctx and ctx.meta_json: + meta = json.loads(ctx.meta_json) or {} + if 'ip' in meta and meta['ip']: + ip = meta['ip'] + if 'ports' in meta and meta['ports']: + port = meta['ports'][0] + r += [f'--web.listen-address={ip}:{port}'] if daemon_type == 'alertmanager': config = get_parm(ctx.config_json) peers = config.get('peers', list()) # type: ignore @@ -2163,7 +2174,6 @@ def create_daemon_dirs(ctx, fsid, daemon_type, daemon_id, uid, gid, if daemon_type in Monitoring.components.keys(): config_json: Dict[str, Any] = get_parm(ctx.config_json) - required_files = Monitoring.components[daemon_type].get('config-json-files', list()) # Set up directories specific to the monitoring component config_dir = '' @@ -2192,10 +2202,14 @@ def create_daemon_dirs(ctx, fsid, daemon_type, daemon_id, uid, gid, makedirs(os.path.join(data_dir_root, config_dir, 'data'), uid, gid, 0o755) # populate the config directory for the component from the config-json - for fname in required_files: - if 'files' in config_json: # type: ignore + if 'files' in config_json: + for fname in config_json['files']: content = dict_get_join(config_json['files'], fname) - with open(os.path.join(data_dir_root, config_dir, fname), 'w') as f: + if os.path.isabs(fname): + fpath = os.path.join(data_dir_root, fname.lstrip(os.path.sep)) + else: + fpath = os.path.join(data_dir_root, config_dir, fname) + with open(fpath, 'w', encoding='utf-8') as f: os.fchown(f.fileno(), uid, gid) os.fchmod(f.fileno(), 0o600) f.write(content) @@ -2267,8 +2281,8 @@ def get_config_and_keyring(ctx): try: with open(ctx.config, 'r') as f: config = f.read() - except FileNotFoundError: - raise Error('config file: %s does not exist' % ctx.config) + except FileNotFoundError as e: + raise Error(e) if 'key' in ctx and ctx.key: keyring = '[%s]\n\tkey = %s\n' % (ctx.name, ctx.key) @@ -2276,8 +2290,8 @@ def get_config_and_keyring(ctx): try: with open(ctx.keyring, 'r') as f: keyring = f.read() - except FileNotFoundError: - raise Error('keyring file: %s does not exist' % ctx.keyring) + except FileNotFoundError as e: + raise Error(e) return config, keyring @@ -2327,19 +2341,20 @@ def get_container_mounts(ctx, fsid, daemon_type, daemon_id, # these do not search for their keyrings in a data directory mounts[data_dir + '/keyring'] = '/etc/ceph/ceph.client.%s.%s.keyring' % (daemon_type, daemon_id) - if daemon_type in ['mon', 'osd']: + if daemon_type in ['mon', 'osd', 'clusterless-ceph-volume']: mounts['/dev'] = '/dev' # FIXME: narrow this down? mounts['/run/udev'] = '/run/udev' - if daemon_type == 'osd': + if daemon_type in ['osd', 'clusterless-ceph-volume']: mounts['/sys'] = '/sys' # for numa.cc, pick_address, cgroups, ... + mounts['/run/lvm'] = '/run/lvm' + mounts['/run/lock/lvm'] = '/run/lock/lvm' + if daemon_type == 'osd': # selinux-policy in the container may not match the host. if HostFacts(ctx).selinux_enabled: selinux_folder = '/var/lib/ceph/%s/selinux' % fsid if not os.path.exists(selinux_folder): os.makedirs(selinux_folder, mode=0o755) mounts[selinux_folder] = '/sys/fs/selinux:ro' - mounts['/run/lvm'] = '/run/lvm' - mounts['/run/lock/lvm'] = '/run/lock/lvm' try: if ctx.shared_ceph_folder: # make easy manager modules/ceph-volume development @@ -2413,7 +2428,9 @@ def get_container(ctx: CephadmContext, entrypoint: str = '' name: str = '' ceph_args: List[str] = [] - envs: List[str] = [] + envs: List[str] = [ + 'TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728', + ] host_network: bool = True if container_args is None: @@ -2540,7 +2557,14 @@ def deploy_daemon(ctx, fsid, daemon_type, daemon_id, c, uid, gid, ports = ports or [] if any([port_in_use(ctx, port) for port in ports]): - raise Error("TCP Port(s) '{}' required for {} already in use".format(','.join(map(str, ports)), daemon_type)) + if daemon_type == 'mgr': + # non-fatal for mgr when we are in mgr_standby_modules=false, but we can't + # tell whether that is the case here. + logger.warning( + f"ceph-mgr TCP port(s) {','.join(map(str, ports))} already in use" + ) + else: + raise Error("TCP Port(s) '{}' required for {} already in use".format(','.join(map(str, ports)), daemon_type)) data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) if reconfig and not os.path.exists(data_dir): @@ -2674,7 +2698,7 @@ def deploy_daemon_units( # cmd data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) with open(data_dir + '/unit.run.new', 'w') as f, \ - open(data_dir + '/unit.meta.new', 'w') as metaf: + open(data_dir + '/unit.meta.new', 'w') as metaf: f.write('set -e\n') if daemon_type in Ceph.daemons: @@ -2710,18 +2734,11 @@ def deploy_daemon_units( memory_limit=ctx.memory_limit, ) _write_container_cmd_to_bash(ctx, f, prestart, 'LVM OSDs use ceph-volume lvm activate') - elif daemon_type == NFSGanesha.daemon_type: - # add nfs to the rados grace db - nfs_ganesha = NFSGanesha.init(ctx, fsid, daemon_id) - prestart = nfs_ganesha.get_rados_grace_container('add') - _write_container_cmd_to_bash(ctx, f, prestart, 'add daemon to rados grace') elif daemon_type == CephIscsi.daemon_type: f.write(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=True)) + '\n') ceph_iscsi = CephIscsi.init(ctx, fsid, daemon_id) tcmu_container = ceph_iscsi.get_tcmu_runner_container() _write_container_cmd_to_bash(ctx, f, tcmu_container, 'iscsi tcmu-runnter container', background=True) - elif daemon_type == Keepalived.daemon_type: - f.write(Keepalived.get_prestart()) _write_container_cmd_to_bash(ctx, f, c, '%s.%s' % (daemon_type, str(daemon_id))) @@ -2763,11 +2780,6 @@ def deploy_daemon_units( daemon_id), ) _write_container_cmd_to_bash(ctx, f, poststop, 'deactivate osd') - elif daemon_type == NFSGanesha.daemon_type: - # remove nfs from the rados grace db - nfs_ganesha = NFSGanesha.init(ctx, fsid, daemon_id) - poststop = nfs_ganesha.get_rados_grace_container('remove') - _write_container_cmd_to_bash(ctx, f, poststop, 'remove daemon from rados grace') elif daemon_type == CephIscsi.daemon_type: # make sure we also stop the tcmu container ceph_iscsi = CephIscsi.init(ctx, fsid, daemon_id) @@ -2785,6 +2797,9 @@ def deploy_daemon_units( os.rename(data_dir + '/unit.image.new', data_dir + '/unit.image') + # sysctl + install_sysctl(ctx, fsid, daemon_type) + # systemd install_base_units(ctx, fsid) unit = get_unit_file(ctx, fsid) @@ -2915,16 +2930,38 @@ class Firewalld(object): def update_firewalld(ctx, daemon_type): # type: (CephadmContext, str) -> None firewall = Firewalld(ctx) - firewall.enable_service_for(daemon_type) + firewall.apply_rules() - fw_ports = [] - if daemon_type in Monitoring.port_map.keys(): - fw_ports.extend(Monitoring.port_map[daemon_type]) # prometheus etc +def install_sysctl(ctx: CephadmContext, fsid: str, daemon_type: str) -> None: + """ + Set up sysctl settings + """ + def _write(conf: Path, lines: List[str]) -> None: + lines = [ + '# created by cephadm', + '', + *lines, + '', + ] + with open(conf, 'w') as f: + f.write('\n'.join(lines)) - firewall.open_ports(fw_ports) - firewall.apply_rules() + conf = Path(ctx.sysctl_dir).joinpath(f'90-ceph-{fsid}-{daemon_type}.conf') + lines: Optional[List] = None + + if daemon_type == 'osd': + lines = OSD.get_sysctl_settings() + elif daemon_type == 'haproxy': + lines = HAproxy.get_sysctl_settings() + elif daemon_type == 'keepalived': + lines = Keepalived.get_sysctl_settings() + + # apply the sysctl settings + if lines: + _write(conf, lines) + call_throws(ctx, ['sysctl', '--system']) def install_base_units(ctx, fsid): @@ -3094,13 +3131,13 @@ class CephContainer: 'run', '--rm', '--ipc=host', + # some containers (ahem, haproxy) override this, but we want a fast + # shutdown always (and, more importantly, a successful exit even if we + # fall back to SIGKILL). + '--stop-signal=SIGTERM', ] if isinstance(self.ctx.container_engine, Podman): - # podman adds the container *name* to /etc/hosts (for 127.0.1.1) - # by default, which makes python's socket.getfqdn() return that - # instead of a valid hostname. - cmd_args.append('--no-hosts') if os.path.exists('/etc/ceph/podman-auth.json'): cmd_args.append('--authfile=/etc/ceph/podman-auth.json') @@ -3167,6 +3204,8 @@ class CephContainer: if self.host_network: cmd_args.append('--net=host') + if self.ctx.no_hosts: + cmd_args.append('--no-hosts') if self.privileged: cmd_args.extend([ '--privileged', @@ -3306,7 +3345,7 @@ def normalize_image_digest(digest): # quay.ceph.io/ceph/ceph -> ceph # docker.io/ubuntu -> no change bits = digest.split('/') - if '.' not in bits[0] or len(bits) < 3: + if '.' not in bits[0] and len(bits) < 3: digest = DEFAULT_REGISTRY + '/' + digest return digest @@ -3438,19 +3477,21 @@ def prepare_mon_addresses( if not ctx.skip_mon_network: # make sure IP is configured locally, and then figure out the # CIDR network + errmsg = f'Cannot infer CIDR network for mon IP `{base_ip}`' for net, ifaces in list_networks(ctx).items(): ips: List[str] = [] for iface, ls in ifaces.items(): ips.extend(ls) - if ipaddress.ip_address(unwrap_ipv6(base_ip)) in \ - [ipaddress.ip_address(ip) for ip in ips]: - mon_network = net - logger.info('Mon IP %s is in CIDR network %s' % (base_ip, - mon_network)) - break + try: + if ipaddress.ip_address(unwrap_ipv6(base_ip)) in \ + [ipaddress.ip_address(ip) for ip in ips]: + mon_network = net + logger.info(f'Mon IP `{base_ip}` is in CIDR network `{mon_network}`') + break + except ValueError as e: + logger.warning(f'{errmsg}: {e}') if not mon_network: - raise Error('Failed to infer CIDR network for mon ip %s; pass ' - '--skip-mon-network to configure it later' % base_ip) + raise Error(f'{errmsg}: pass --skip-mon-network to configure it later') return (addr_arg, ipv6, mon_network) @@ -3832,6 +3873,7 @@ def prepare_bootstrap_config( cp.set('global', 'fsid', fsid) cp.set('global', 'mon_host', mon_addr) cp.set('global', 'container_image', image) + if not cp.has_section('mon'): cp.add_section('mon') if ( @@ -3839,6 +3881,30 @@ def prepare_bootstrap_config( and not cp.has_option('mon', 'auth allow insecure global id reclaim') ): cp.set('mon', 'auth_allow_insecure_global_id_reclaim', 'false') + + if ctx.single_host_defaults: + logger.info('Adjusting default settings to suit single-host cluster...') + # replicate across osds, not hosts + if ( + not cp.has_option('global', 'osd_crush_choose_leaf_type') + and not cp.has_option('global', 'osd crush choose leaf type') + ): + cp.set('global', 'osd_crush_choose_leaf_type', '0') + # replica 2x + if ( + not cp.has_option('global', 'osd_pool_default_size') + and not cp.has_option('global', 'osd pool default size') + ): + cp.set('global', 'osd_pool_default_size', '2') + # disable mgr standby modules (so we can colocate multiple mgrs on one host) + if not cp.has_section('mgr'): + cp.add_section('mgr') + if ( + not cp.has_option('mgr', 'mgr_standby_modules') + and not cp.has_option('mgr', 'mgr standby modules') + ): + cp.set('mgr', 'mgr_standby_modules', 'false') + cpf = StringIO() cp.write(cpf) config = cpf.getvalue() @@ -3931,6 +3997,8 @@ def command_bootstrap(ctx): except PermissionError: raise Error(f'Unable to create {dirname} due to permissions failure. Retry with root, or sudo or preallocate the directory.') + (user_conf, _) = get_config_and_keyring(ctx) + if not ctx.skip_prepare_host: command_prepare_host(ctx) else: @@ -3938,12 +4006,15 @@ def command_bootstrap(ctx): # initial vars fsid = ctx.fsid or make_fsid() + if not is_fsid(fsid): + raise Error('not an fsid: %s' % fsid) + logger.info('Cluster fsid: %s' % fsid) + hostname = get_hostname() if '.' in hostname and not ctx.allow_fqdn_hostname: raise Error('hostname is a fully qualified domain name (%s); either fix (e.g., "sudo hostname %s" or similar) or pass --allow-fqdn-hostname' % (hostname, hostname.split('.')[0])) mon_id = ctx.mon_id or hostname mgr_id = ctx.mgr_id or generate_service_id() - logger.info('Cluster fsid: %s' % fsid) lock = FileLock(ctx, fsid) lock.acquire() @@ -3958,14 +4029,15 @@ def command_bootstrap(ctx): image_ver = CephContainer(ctx, ctx.image, 'ceph', ['--version']).run().strip() logger.info(f'Ceph version: {image_ver}') - image_release = image_ver.split()[4] - if ( - not ctx.allow_mismatched_release - and image_release not in [DEFAULT_IMAGE_RELEASE, LATEST_STABLE_RELEASE] - ): - raise Error( - f'Container release {image_release} != cephadm release {DEFAULT_IMAGE_RELEASE}; please use matching version of cephadm (pass --allow-mismatched-release to continue anyway)' - ) + + if not ctx.allow_mismatched_release: + image_release = image_ver.split()[4] + if image_release not in \ + [DEFAULT_IMAGE_RELEASE, LATEST_STABLE_RELEASE]: + raise Error( + f'Container release {image_release} != cephadm release {DEFAULT_IMAGE_RELEASE};' + ' please use matching version of cephadm (pass --allow-mismatched-release to continue anyway)' + ) logger.info('Extracting ceph user uid/gid from container image...') (uid, gid) = extract_uid_gid(ctx) @@ -4025,14 +4097,18 @@ def command_bootstrap(ctx): # create mgr create_mgr(ctx, uid, gid, fsid, mgr_id, mgr_key, config, cli) - def json_loads_retry(cli_func): - for sleep_secs in [1, 4, 4]: - try: - return json.loads(cli_func()) - except json.JSONDecodeError: - logger.debug('Invalid JSON. Retrying in %s seconds...' % sleep_secs) - time.sleep(sleep_secs) - return json.loads(cli_func()) + if user_conf: + # user given config settings were already assimilated earlier + # but if the given settings contained any attributes in + # the mgr (e.g. mgr/cephadm/container_image_prometheus) + # they don't seem to be stored if there isn't a mgr yet. + # Since re-assimilating the same conf settings should be + # idempotent we can just do it again here. + with tempfile.NamedTemporaryFile(buffering=0) as tmp: + tmp.write(user_conf.encode('utf-8')) + cli(['config', 'assimilate-conf', + '-i', '/var/lib/ceph/user.conf'], + {tmp.name: '/var/lib/ceph/user.conf:z'}) # wait for mgr to restart (after enabling a module) def wait_for_mgr_restart(): @@ -4096,6 +4172,14 @@ def command_bootstrap(ctx): if not ctx.skip_dashboard: prepare_dashboard(ctx, uid, gid, cli, wait_for_mgr_restart) + if ctx.output_config == '/etc/ceph/ceph.conf' and not ctx.skip_admin_label: + logger.info('Enabling client.admin keyring and conf on hosts with "admin" label') + try: + cli(['orch', 'client-keyring', 'set', 'client.admin', 'label:_admin']) + cli(['orch', 'host', 'label', 'add', get_hostname(), '_admin']) + except Exception: + logger.info('Unable to set up "admin" label; assuming older version of Ceph') + if ctx.apply_spec: logger.info('Applying %s to cluster' % ctx.apply_spec) @@ -4244,9 +4328,6 @@ def command_deploy(ctx): elif daemon_type in Monitoring.components: # monitoring daemon - prometheus, grafana, alertmanager, node-exporter # Default Checks - if not ctx.reconfig and not redeploy: - daemon_ports.extend(Monitoring.port_map[daemon_type]) - # make sure provided config-json is sufficient config = get_parm(ctx.config_json) # type: ignore required_files = Monitoring.components[daemon_type].get('config-json-files', list()) @@ -4267,8 +4348,8 @@ def command_deploy(ctx): ports=daemon_ports) elif daemon_type == NFSGanesha.daemon_type: - if not ctx.reconfig and not redeploy: - daemon_ports.extend(NFSGanesha.port_map.values()) + if not ctx.reconfig and not redeploy and not daemon_ports: + daemon_ports = list(NFSGanesha.port_map.values()) config, keyring = get_config_and_keyring(ctx) # TODO: extract ganesha uid/gid (997, 994) ? @@ -4427,6 +4508,10 @@ def command_shell(ctx): os.path.join(home, f)) mounts[home] = '/root' + for i in ctx.volume: + a, b = i.split(':', 1) + mounts[a] = b + c = CephContainer( ctx, image=ctx.image, @@ -4510,8 +4595,8 @@ def command_ceph_volume(ctx): privileged=True, volume_mounts=mounts, ) - verbosity = CallVerbosity.VERBOSE if ctx.log_output else CallVerbosity.VERBOSE_ON_FAILURE - out, err, code = call_throws(ctx, c.run_cmd(), verbosity=verbosity) + + out, err, code = call_throws(ctx, c.run_cmd()) if not code: print(out) @@ -5342,6 +5427,68 @@ def command_rm_daemon(ctx): ################################## +def _zap(ctx, what): + mounts = get_container_mounts(ctx, ctx.fsid, 'clusterless-ceph-volume', None) + c = CephContainer( + ctx, + image=ctx.image, + entrypoint='/usr/sbin/ceph-volume', + envs=ctx.env, + args=['lvm', 'zap', '--destroy', what], + privileged=True, + volume_mounts=mounts, + ) + logger.info(f'Zapping {what}...') + out, err, code = call_throws(ctx, c.run_cmd()) + + +@infer_image +def _zap_osds(ctx): + # assume fsid lock already held + + # list + mounts = get_container_mounts(ctx, ctx.fsid, 'clusterless-ceph-volume', None) + c = CephContainer( + ctx, + image=ctx.image, + entrypoint='/usr/sbin/ceph-volume', + envs=ctx.env, + args=['inventory', '--format', 'json'], + privileged=True, + volume_mounts=mounts, + ) + out, err, code = call_throws(ctx, c.run_cmd()) + if code: + raise Error('failed to list osd inventory') + try: + ls = json.loads(out) + except ValueError as e: + raise Error(f'Invalid JSON in ceph-volume inventory: {e}') + + for i in ls: + matches = [lv.get('cluster_fsid') == ctx.fsid for lv in i.get('lvs', [])] + if any(matches) and all(matches): + _zap(ctx, i.get('path')) + elif any(matches): + lv_names = [lv['name'] for lv in i.get('lvs', [])] + # TODO: we need to map the lv_names back to device paths (the vg + # id isn't part of the output here!) + logger.warning(f'Not zapping LVs (not implemented): {lv_names}') + + +def command_zap_osds(ctx): + if not ctx.force: + raise Error('must pass --force to proceed: ' + 'this command may destroy precious data!') + + lock = FileLock(ctx, ctx.fsid) + lock.acquire() + + _zap_osds(ctx) + +################################## + + def command_rm_cluster(ctx): # type: (CephadmContext) -> None if not ctx.force: @@ -5378,11 +5525,15 @@ def command_rm_cluster(ctx): call(ctx, ['systemctl', 'stop', slice_name], verbosity=CallVerbosity.DEBUG) + # osds? + if ctx.zap_osds: + _zap_osds(ctx) + # rm units - call_throws(ctx, ['rm', '-f', ctx.unit_dir + # noqa: W504 - '/ceph-%s@.service' % ctx.fsid]) - call_throws(ctx, ['rm', '-f', ctx.unit_dir + # noqa: W504 - '/ceph-%s.target' % ctx.fsid]) + call_throws(ctx, ['rm', '-f', ctx.unit_dir + + '/ceph-%s@.service' % ctx.fsid]) + call_throws(ctx, ['rm', '-f', ctx.unit_dir + + '/ceph-%s.target' % ctx.fsid]) call_throws(ctx, ['rm', '-rf', ctx.unit_dir + '/ceph-%s.target.wants' % ctx.fsid]) # rm data @@ -5391,12 +5542,21 @@ def command_rm_cluster(ctx): if not ctx.keep_logs: # rm logs call_throws(ctx, ['rm', '-rf', ctx.log_dir + '/' + ctx.fsid]) - call_throws(ctx, ['rm', '-rf', ctx.log_dir + # noqa: W504 - '/*.wants/ceph-%s@*' % ctx.fsid]) + call_throws(ctx, ['rm', '-rf', ctx.log_dir + + '/*.wants/ceph-%s@*' % ctx.fsid]) # rm logrotate config call_throws(ctx, ['rm', '-f', ctx.logrotate_dir + '/ceph-%s' % ctx.fsid]) + # rm cephadm logrotate config if last cluster on host + if not os.listdir(ctx.data_dir): + call_throws(ctx, ['rm', '-f', ctx.logrotate_dir + '/cephadm']) + + # rm sysctl settings + sysctl_dir = Path(ctx.sysctl_dir) + for p in sysctl_dir.glob(f'90-ceph-{ctx.fsid}-*.conf'): + p.unlink() + # clean up config, keyring, and pub key files files = ['/etc/ceph/ceph.conf', '/etc/ceph/ceph.pub', '/etc/ceph/ceph.client.admin.keyring'] @@ -5410,6 +5570,7 @@ def command_rm_cluster(ctx): if os.path.exists(files[n]): os.remove(files[n]) + ################################## @@ -5608,9 +5769,9 @@ class Packager(object): if self.ctx.gpg_url: return self.ctx.gpg_url if self.stable or self.version: - return 'https://download.ceph.com/keys/release.asc', 'release' + return 'https://download.ceph.com/keys/release.gpg', 'release' else: - return 'https://download.ceph.com/keys/autobuild.asc', 'autobuild' + return 'https://download.ceph.com/keys/autobuild.gpg', 'autobuild' def enable_service(self, service): """ @@ -5648,8 +5809,8 @@ class Apt(Packager): logger.error('failed to fetch GPG repo key from %s: %s' % ( url, err)) raise Error('failed to fetch GPG key') - key = response.read().decode('utf-8') - with open('/etc/apt/trusted.gpg.d/ceph.%s.gpg' % name, 'w') as f: + key = response.read() + with open('/etc/apt/trusted.gpg.d/ceph.%s.gpg' % name, 'wb') as f: f.write(key) if self.version: @@ -5666,6 +5827,8 @@ class Apt(Packager): with open(self.repo_path(), 'w') as f: f.write(content) + self.update() + def rm_repo(self): for name in ['autobuild', 'release']: p = '/etc/apt/trusted.gpg.d/ceph.%s.gpg' % name @@ -5683,11 +5846,15 @@ class Apt(Packager): logger.info('Installing packages %s...' % ls) call_throws(self.ctx, ['apt-get', 'install', '-y'] + ls) + def update(self): + logger.info('Updating package list...') + call_throws(self.ctx, ['apt-get', 'update']) + def install_podman(self): if self.distro == 'ubuntu': logger.info('Setting up repo for podman...') self.add_kubic_repo() - call_throws(self.ctx, ['apt-get', 'update']) + self.update() logger.info('Attempting podman install...') try: @@ -5745,6 +5912,7 @@ class YumDnf(Packager): 'centos': ('centos', 'el'), 'rhel': ('centos', 'el'), 'scientific': ('centos', 'el'), + 'rocky': ('centos', 'el'), 'fedora': ('fedora', 'fc'), } @@ -5832,6 +6000,13 @@ class YumDnf(Packager): self.distro_code) def add_repo(self): + if self.distro_code.startswith('fc'): + raise Error('Ceph team does not build Fedora specific packages and therefore cannot add repos for this distro') + if self.distro_code == 'el7': + if self.stable and self.stable >= 'pacific': + raise Error('Ceph does not support pacific or later for this version of this linux distro and therefore cannot add a repo for it') + if self.version and self.version.split('.')[0] >= '16': + raise Error('Ceph does not support 16.y.z or later for this version of this linux distro and therefore cannot add a repo for it') if self.stable or self.version: content = '' for n, t in { @@ -5998,12 +6173,16 @@ def command_add_repo(ctx: CephadmContext): (x, y, z) = ctx.version.split('.') except Exception: raise Error('version must be in the form x.y.z (e.g., 15.2.0)') + if ctx.release: + # Pacific =/= pacific in this case, set to undercase to avoid confision + ctx.release = ctx.release.lower() pkg = create_packager(ctx, stable=ctx.release, version=ctx.version, branch=ctx.dev, commit=ctx.dev_commit) pkg.add_repo() + logger.info('Completed adding repo.') def command_rm_repo(ctx: CephadmContext): @@ -6501,6 +6680,8 @@ class HostFacts(): security['description'] = 'AppArmor: Enabled' try: profiles = read_file(['/sys/kernel/security/apparmor/profiles']) + if len(profiles) == 0: + return {} except OSError: pass else: @@ -6580,16 +6761,6 @@ def command_gather_facts(ctx: CephadmContext): host = HostFacts(ctx) print(host.dump()) -################################## - - -def command_verify_prereqs(ctx: CephadmContext): - if ctx.service_type == 'haproxy' or ctx.service_type == 'keepalived': - out, err, code = call( - ctx, ['sysctl', '-n', 'net.ipv4.ip_nonlocal_bind'] - ) - if out.strip() != '1': - raise Error('net.ipv4.ip_nonlocal_bind not set to 1') ################################## @@ -6671,12 +6842,14 @@ class CephadmDaemonHandler(BaseHTTPRequestHandler): The token is installed at deployment time and must be provided to ensure we only respond to callers who know our token i.e. mgr """ + def wrapper(self, *args, **kwargs): auth = self.headers.get('Authorization', None) if auth != 'Bearer ' + self.server.token: self.send_error(401) return f(self, *args, **kwargs) + return wrapper def _help_page(self): @@ -6950,7 +7123,6 @@ class CephadmDaemon(): # expects to use self.ctx.command = 'inventory --format=json'.split() self.ctx.fsid = self.fsid - self.ctx.log_output = False ctr = 0 exception_encountered = False @@ -7352,6 +7524,10 @@ def _get_parser(): '--logrotate-dir', default=LOGROTATE_DIR, help='location of logrotate configuration files') + parser.add_argument( + '--sysctl-dir', + default=SYSCTL_DIR, + help='location of sysctl configuration files') parser.add_argument( '--unit-dir', default=UNIT_DIR, @@ -7487,6 +7663,10 @@ def _get_parser(): '--keep-logs', action='store_true', help='do not remove log files') + parser_rm_cluster.add_argument( + '--zap-osds', + action='store_true', + help='zap OSD devices for this cluster') parser_run = subparsers.add_parser( 'run', help='run a ceph daemon, in a container, in the foreground') @@ -7527,9 +7707,18 @@ def _get_parser(): action='append', default=[], help='set environment variable') + parser_shell.add_argument( + '--volume', '-v', + action='append', + default=[], + help='set environment variable') parser_shell.add_argument( 'command', nargs=argparse.REMAINDER, help='command (optional)') + parser_shell.add_argument( + '--no-hosts', + action='store_true', + help='dont pass /etc/hosts through to the container') parser_enter = subparsers.add_parser( 'enter', help='run an interactive shell inside a running daemon container') @@ -7560,15 +7749,22 @@ def _get_parser(): parser_ceph_volume.add_argument( '--keyring', '-k', help='ceph.keyring to pass through to the container') - parser_ceph_volume.add_argument( - '--log-output', - action='store_true', - default=True, - help='suppress ceph volume output from the log') parser_ceph_volume.add_argument( 'command', nargs=argparse.REMAINDER, help='command') + parser_zap_osds = subparsers.add_parser( + 'zap-osds', help='zap all OSDs associated with a particular fsid') + parser_zap_osds.set_defaults(func=command_zap_osds) + parser_zap_osds.add_argument( + '--fsid', + required=True, + help='cluster FSID') + parser_zap_osds.add_argument( + '--force', + action='store_true', + help='proceed, even though this may destroy valuable data') + parser_unit = subparsers.add_parser( 'unit', help="operate on the daemon's systemd unit") parser_unit.set_defaults(func=command_unit) @@ -7633,6 +7829,10 @@ def _get_parser(): parser_bootstrap.add_argument( '--output-pub-ssh-key', help="location to write the cluster's public SSH key") + parser_bootstrap.add_argument( + '--skip-admin-label', + action='store_true', + help='do not create admin label for ceph.conf and client.admin keyring distribution') parser_bootstrap.add_argument( '--skip-ssh', action='store_true', @@ -7764,6 +7964,10 @@ def _get_parser(): parser_bootstrap.add_argument( '--cluster-network', help='subnet to use for cluster replication, recovery and heartbeats (in CIDR notation network/mask)') + parser_bootstrap.add_argument( + '--single-host-defaults', + action='store_true', + help='adjust configuration defaults to suit a single-host cluster') parser_deploy = subparsers.add_parser( 'deploy', help='deploy a daemon') @@ -7929,15 +8133,6 @@ def _get_parser(): help='Maintenance action - enter maintenance, or exit maintenance') parser_maintenance.set_defaults(func=command_maintenance) - parser_verify_prereqs = subparsers.add_parser( - 'verify-prereqs', - help='verify system prerequisites for a given service are met on this host') - parser_verify_prereqs.set_defaults(func=command_verify_prereqs) - parser_verify_prereqs.add_argument( - '--daemon-type', - required=True, - help='service type of service to whose prereqs will be checked') - return parser @@ -7962,18 +8157,15 @@ def _parse_args(av): return args -def cephadm_init_ctx(args: List[str]) -> Optional[CephadmContext]: - +def cephadm_init_ctx(args: List[str]) -> CephadmContext: ctx = CephadmContext() ctx.set_args(_parse_args(args)) return ctx -def cephadm_init(args: List[str]) -> Optional[CephadmContext]: - +def cephadm_init(args: List[str]) -> CephadmContext: global logger ctx = cephadm_init_ctx(args) - assert ctx is not None # Logger configuration if not os.path.exists(LOG_DIR): @@ -7981,15 +8173,23 @@ def cephadm_init(args: List[str]) -> Optional[CephadmContext]: dictConfig(logging_config) logger = logging.getLogger() + if not os.path.exists(ctx.logrotate_dir + '/cephadm'): + with open(ctx.logrotate_dir + '/cephadm', 'w') as f: + f.write("""# created by cephadm +/var/log/ceph/cephadm.log { + rotate 7 + daily + compress + missingok + notifempty +} +""") + if ctx.verbose: for handler in logger.handlers: if handler.name == 'console': handler.setLevel(logging.DEBUG) - if not ctx.has_function(): - sys.stderr.write('No command specified; pass -h or --help for usage\n') - return None - return ctx @@ -8004,14 +8204,15 @@ def main(): av = sys.argv[1:] ctx = cephadm_init(av) - if not ctx: # error, exit + if not ctx.has_function(): + sys.stderr.write('No command specified; pass -h or --help for usage\n') sys.exit(1) try: # podman or docker? ctx.container_engine = find_container_engine(ctx) if ctx.func not in \ - [command_check_host, command_prepare_host, command_add_repo]: + [command_check_host, command_prepare_host, command_add_repo, command_install]: check_container_engine(ctx) # command handler r = ctx.func(ctx) diff --git a/ceph/src/cephadm/tests/fixtures.py b/ceph/src/cephadm/tests/fixtures.py index a3f462539..2f07d6034 100644 --- a/ceph/src/cephadm/tests/fixtures.py +++ b/ceph/src/cephadm/tests/fixtures.py @@ -1,16 +1,32 @@ - import mock -from mock import patch -import pytest - import os +import pytest import time -with patch('builtins.open', create=True): +from contextlib import contextmanager +from pyfakefs import fake_filesystem + +from typing import Callable, Dict, List, Optional + + +with mock.patch('builtins.open', create=True): from importlib.machinery import SourceFileLoader cd = SourceFileLoader('cephadm', 'cephadm').load_module() +def mock_docker(): + docker = mock.Mock(cd.Docker) + docker.path = '/usr/bin/docker' + return docker + + +def mock_podman(): + podman = mock.Mock(cd.Podman) + podman.path = '/usr/bin/podman' + podman.version = (2, 1, 0) + return podman + + def _daemon_path(): return os.getcwd() @@ -40,3 +56,56 @@ def exporter(): exporter = cd.CephadmDaemon(ctx, fsid='foobar', daemon_id='test') assert exporter.token == 'MyAccessToken' yield exporter + + +@pytest.fixture() +def cephadm_fs( + fs: fake_filesystem.FakeFilesystem, +): + """ + use pyfakefs to stub filesystem calls + """ + uid = os.getuid() + gid = os.getgid() + + with mock.patch('os.fchown'), \ + mock.patch('cephadm.extract_uid_gid', return_value=(uid, gid)): + + fs.create_dir(cd.DATA_DIR) + fs.create_dir(cd.LOG_DIR) + fs.create_dir(cd.LOCK_DIR) + fs.create_dir(cd.LOGROTATE_DIR) + fs.create_dir(cd.UNIT_DIR) + + yield fs + + +@contextmanager +def with_cephadm_ctx( + cmd: List[str], + container_engine: Callable = mock_podman(), + list_networks: Optional[Dict[str,Dict[str,List[str]]]] = None, + hostname: Optional[str] = None, +): + """ + :param cmd: cephadm command argv + :param container_engine: container engine mock (podman or docker) + :param list_networks: mock 'list-networks' return + :param hostname: mock 'socket.gethostname' return + """ + if not list_networks: + list_networks = {} + if not hostname: + hostname = 'host1' + + with mock.patch('cephadm.get_parm'), \ + mock.patch('cephadm.attempt_bind'), \ + mock.patch('cephadm.call', return_value=('', '', 0)), \ + mock.patch('cephadm.find_executable', return_value='foo'), \ + mock.patch('cephadm.is_available', return_value=True), \ + mock.patch('cephadm.json_loads_retry', return_value={'epoch' : 1}), \ + mock.patch('cephadm.list_networks', return_value=list_networks), \ + mock.patch('socket.gethostname', return_value=hostname): + ctx: cd.CephadmContext = cd.cephadm_init_ctx(cmd) + ctx.container_engine = container_engine + yield ctx diff --git a/ceph/src/cephadm/tests/test_cephadm.py b/ceph/src/cephadm/tests/test_cephadm.py index 15b6efcc4..42853fb6b 100644 --- a/ceph/src/cephadm/tests/test_cephadm.py +++ b/ceph/src/cephadm/tests/test_cephadm.py @@ -1,47 +1,43 @@ # type: ignore -from typing import List, Optional + +import errno import mock -from mock import patch, call import os +import pytest +import socket import sys -import unittest -import threading import time -import errno -import socket +import threading +import unittest + from http.server import HTTPServer from urllib.request import Request, urlopen from urllib.error import HTTPError -import pytest +from typing import List, Optional -from .fixtures import exporter +from .fixtures import ( + cephadm_fs, + exporter, + mock_docker, + mock_podman, + with_cephadm_ctx, +) -with patch('builtins.open', create=True): + +with mock.patch('builtins.open', create=True): from importlib.machinery import SourceFileLoader cd = SourceFileLoader('cephadm', 'cephadm').load_module() -class TestCephAdm(object): - - @staticmethod - def mock_docker(): - docker = mock.Mock(cd.Docker) - docker.path = '/usr/bin/docker' - return docker - @staticmethod - def mock_podman(): - podman = mock.Mock(cd.Podman) - podman.path = '/usr/bin/podman' - podman.version = (2, 1, 0) - return podman +class TestCephAdm(object): def test_docker_unit_file(self): ctx = mock.Mock() - ctx.container_engine = self.mock_docker() + ctx.container_engine = mock_docker() r = cd.get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9') assert 'Requires=docker.service' in r - ctx.container_engine = self.mock_podman() + ctx.container_engine = mock_podman() r = cd.get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9') assert 'Requires=docker.service' not in r @@ -58,8 +54,8 @@ class TestCephAdm(object): for side_effect, expected_exception in ( (os_error(errno.EADDRINUSE), cd.PortOccupiedError), - (os_error(errno.EAFNOSUPPORT), OSError), - (os_error(errno.EADDRNOTAVAIL), OSError), + (os_error(errno.EAFNOSUPPORT), cd.Error), + (os_error(errno.EADDRNOTAVAIL), cd.Error), (None, None), ): _socket = mock.Mock() @@ -107,7 +103,7 @@ class TestCephAdm(object): except: assert False else: - assert _socket.call_args == call(address_family, socket.SOCK_STREAM) + assert _socket.call_args == mock.call(address_family, socket.SOCK_STREAM) @mock.patch('socket.socket') @mock.patch('cephadm.logger') @@ -126,8 +122,8 @@ class TestCephAdm(object): ): for side_effect, expected_exception in ( (os_error(errno.EADDRINUSE), cd.PortOccupiedError), - (os_error(errno.EADDRNOTAVAIL), OSError), - (os_error(errno.EAFNOSUPPORT), OSError), + (os_error(errno.EADDRNOTAVAIL), cd.Error), + (os_error(errno.EAFNOSUPPORT), cd.Error), (None, None), ): mock_socket_obj = mock.Mock() @@ -410,19 +406,17 @@ default proto ra metric 100 # test normal valid login with url, username and password specified call_throws.return_value = '', '', 0 - ctx: Optional[cd.CephadmContext] = cd.cephadm_init_ctx( + ctx: cd.CephadmContext = cd.cephadm_init_ctx( ['registry-login', '--registry-url', 'sample-url', '--registry-username', 'sample-user', '--registry-password', 'sample-pass']) - ctx.container_engine = self.mock_docker() - assert ctx + ctx.container_engine = mock_docker() retval = cd.command_registry_login(ctx) assert retval == 0 # test bad login attempt with invalid arguments given - ctx: Optional[cd.CephadmContext] = cd.cephadm_init_ctx( + ctx: cd.CephadmContext = cd.cephadm_init_ctx( ['registry-login', '--registry-url', 'bad-args-url']) - assert ctx with pytest.raises(Exception) as e: assert cd.command_registry_login(ctx) assert str(e.value) == ('Invalid custom registry arguments received. To login to a custom registry include ' @@ -430,18 +424,16 @@ default proto ra metric 100 # test normal valid login with json file get_parm.return_value = {"url": "sample-url", "username": "sample-username", "password": "sample-password"} - ctx: Optional[cd.CephadmContext] = cd.cephadm_init_ctx( + ctx: cd.CephadmContext = cd.cephadm_init_ctx( ['registry-login', '--registry-json', 'sample-json']) - ctx.container_engine = self.mock_docker() - assert ctx + ctx.container_engine = mock_docker() retval = cd.command_registry_login(ctx) assert retval == 0 # test bad login attempt with bad json file get_parm.return_value = {"bad-json": "bad-json"} - ctx: Optional[cd.CephadmContext] = cd.cephadm_init_ctx( + ctx: cd.CephadmContext = cd.cephadm_init_ctx( ['registry-login', '--registry-json', 'sample-json']) - assert ctx with pytest.raises(Exception) as e: assert cd.command_registry_login(ctx) assert str(e.value) == ("json provided for custom registry login did not include all necessary fields. " @@ -454,11 +446,10 @@ default proto ra metric 100 # test login attempt with valid arguments where login command fails call_throws.side_effect = Exception - ctx: Optional[cd.CephadmContext] = cd.cephadm_init_ctx( + ctx: cd.CephadmContext = cd.cephadm_init_ctx( ['registry-login', '--registry-url', 'sample-url', '--registry-username', 'sample-user', '--registry-password', 'sample-pass']) - assert ctx with pytest.raises(Exception) as e: cd.command_registry_login(ctx) assert str(e.value) == "Failed to login to custom registry @ sample-url as sample-user with given password" @@ -524,6 +515,12 @@ docker.io/ceph/daemon-base:octopus image = cd._filter_last_local_ceph_image(out) assert image == 'docker.io/ceph/ceph:v15.2.5' + def test_normalize_image_digest(self): + s = 'myhostname:5000/ceph/ceph@sha256:753886ad9049004395ae990fbb9b096923b5a518b819283141ee8716ddf55ad1' + assert cd.normalize_image_digest(s) == s + + s = 'ceph/ceph:latest' + assert cd.normalize_image_digest(s) == f'{cd.DEFAULT_REGISTRY}/{s}' class TestCustomContainer(unittest.TestCase): cc: cd.CustomContainer @@ -959,3 +956,211 @@ class TestMonitoring(object): _call.return_value = '', '{}, version 0.16.1'.format(daemon_type.replace('-', '_')), 0 version = cd.Monitoring.get_version(ctx, 'container_id', daemon_type) assert version == '0.16.1' + + @mock.patch('cephadm.os.fchown') + @mock.patch('cephadm.get_parm') + @mock.patch('cephadm.makedirs') + @mock.patch('cephadm.open') + @mock.patch('cephadm.make_log_dir') + @mock.patch('cephadm.make_data_dir') + def test_create_daemon_dirs_prometheus(self, make_data_dir, make_log_dir, _open, makedirs, + get_parm, fchown): + """ + Ensures the required and optional files given in the configuration are + created and mapped correctly inside the container. Tests absolute and + relative file paths given in the configuration. + """ + + fsid = 'aaf5a720-13fe-4a3b-82b9-2d99b7fd9704' + daemon_type = 'prometheus' + uid, gid = 50, 50 + daemon_id = 'home' + ctx = mock.Mock() + ctx.data_dir = '/somedir' + files = { + 'files': { + 'prometheus.yml': 'foo', + '/etc/prometheus/alerting/ceph_alerts.yml': 'bar' + } + } + get_parm.return_value = files + + cd.create_daemon_dirs(ctx, + fsid, + daemon_type, + daemon_id, + uid, + gid, + config=None, + keyring=None) + + prefix = '{data_dir}/{fsid}/{daemon_type}.{daemon_id}'.format( + data_dir=ctx.data_dir, + fsid=fsid, + daemon_type=daemon_type, + daemon_id=daemon_id + ) + assert _open.call_args_list == [ + mock.call('{}/etc/prometheus/prometheus.yml'.format(prefix), 'w', + encoding='utf-8'), + mock.call('{}/etc/prometheus/alerting/ceph_alerts.yml'.format(prefix), 'w', + encoding='utf-8'), + ] + assert mock.call().__enter__().write('foo') in _open.mock_calls + assert mock.call().__enter__().write('bar') in _open.mock_calls + + +class TestBootstrap(object): + + @staticmethod + def _get_cmd(*args): + return [ + 'bootstrap', + '--allow-mismatched-release', + '--skip-prepare-host', + '--skip-dashboard', + *args, + ] + + def test_config(self, cephadm_fs): + conf_file = 'foo' + cmd = self._get_cmd( + '--mon-ip', '192.168.1.1', + '--skip-mon-network', + '--config', conf_file, + ) + + with with_cephadm_ctx(cmd) as ctx: + msg = r'No such file or directory' + with pytest.raises(cd.Error, match=msg): + cd.command_bootstrap(ctx) + + cephadm_fs.create_file(conf_file) + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_bootstrap(ctx) + assert retval == 0 + + def test_no_mon_addr(self, cephadm_fs): + cmd = self._get_cmd() + with with_cephadm_ctx(cmd) as ctx: + msg = r'must specify --mon-ip or --mon-addrv' + with pytest.raises(cd.Error, match=msg): + cd.command_bootstrap(ctx) + + def test_skip_mon_network(self, cephadm_fs): + cmd = self._get_cmd('--mon-ip', '192.168.1.1') + + with with_cephadm_ctx(cmd, list_networks={}) as ctx: + msg = r'--skip-mon-network' + with pytest.raises(cd.Error, match=msg): + cd.command_bootstrap(ctx) + + cmd += ['--skip-mon-network'] + with with_cephadm_ctx(cmd, list_networks={}) as ctx: + retval = cd.command_bootstrap(ctx) + assert retval == 0 + + @pytest.mark.parametrize('mon_ip, list_networks, result', + [ + # IPv4 + ( + 'eth0', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + False, + ), + ( + '0.0.0.0', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + False, + ), + ( + '192.168.1.0', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + False, + ), + ( + '192.168.1.1', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + True, + ), + ( + '192.168.1.1:1234', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + True, + ), + # IPv6 + ( + '::', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + False, + ), + ( + '::ffff:192.168.1.0', + {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}}, + False, + ), + ( + '::ffff:192.168.1.1', + {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}}, + True, + ), + ( + '::ffff:c0a8:101', + {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}}, + True, + ), + ( + '0000:0000:0000:0000:0000:FFFF:C0A8:0101', + {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}}, + True, + ), + ]) + def test_mon_ip(self, mon_ip, list_networks, result, cephadm_fs): + cmd = self._get_cmd('--mon-ip', mon_ip) + if not result: + with with_cephadm_ctx(cmd, list_networks=list_networks) as ctx: + msg = r'--skip-mon-network' + with pytest.raises(cd.Error, match=msg): + cd.command_bootstrap(ctx) + else: + with with_cephadm_ctx(cmd, list_networks=list_networks) as ctx: + retval = cd.command_bootstrap(ctx) + assert retval == 0 + + def test_allow_fqdn_hostname(self, cephadm_fs): + hostname = 'foo.bar' + cmd = self._get_cmd( + '--mon-ip', '192.168.1.1', + '--skip-mon-network', + ) + + with with_cephadm_ctx(cmd, hostname=hostname) as ctx: + msg = r'--allow-fqdn-hostname' + with pytest.raises(cd.Error, match=msg): + cd.command_bootstrap(ctx) + + cmd += ['--allow-fqdn-hostname'] + with with_cephadm_ctx(cmd, hostname=hostname) as ctx: + retval = cd.command_bootstrap(ctx) + assert retval == 0 + + @pytest.mark.parametrize('fsid, err', + [ + ('', None), + ('00000000-0000-0000-0000-0000deadbeef', None), + ('00000000-0000-0000-0000-0000deadbeez', 'not an fsid'), + ]) + def test_fsid(self, fsid, err, cephadm_fs): + cmd = self._get_cmd( + '--mon-ip', '192.168.1.1', + '--skip-mon-network', + '--fsid', fsid, + ) + + with with_cephadm_ctx(cmd) as ctx: + if err: + with pytest.raises(cd.Error, match=err): + cd.command_bootstrap(ctx) + else: + retval = cd.command_bootstrap(ctx) + assert retval == 0 diff --git a/ceph/src/cephadm/tox.ini b/ceph/src/cephadm/tox.ini index cbe71cae8..6f25b4008 100644 --- a/ceph/src/cephadm/tox.ini +++ b/ceph/src/cephadm/tox.ini @@ -1,6 +1,11 @@ [tox] -envlist = py3, mypy, flake8 -skipsdist=true +envlist = + py3 + mypy + fix + flake8 +skipsdist = true +requires = cython [flake8] max-line-length = 100 @@ -17,11 +22,21 @@ exclude = .eggs statistics = True +[autopep8] +addopts = + --max-line-length {[flake8]max-line-length} + --ignore "{[flake8]ignore}" + --exclude "{[flake8]exclude}" + --in-place + --recursive + --ignore-local-config + [testenv] skip_install=true deps = - pytest + pyfakefs mock + pytest commands=pytest {posargs} [testenv:mypy] @@ -29,6 +44,14 @@ basepython = python3 deps = mypy==0.790 commands = mypy --config-file ../mypy.ini {posargs:cephadm} +[testenv:fix] +basepython = python3 +deps = + autopep8 +commands = + python --version + autopep8 {[autopep8]addopts} {posargs: cephadm} + [testenv:flake8] basepython = python3 deps = diff --git a/ceph/src/client/Client.cc b/ceph/src/client/Client.cc index a94876881..7d052bad8 100644 --- a/ceph/src/client/Client.cc +++ b/ceph/src/client/Client.cc @@ -145,6 +145,10 @@ #define DEBUG_GETATTR_CAPS (CEPH_CAP_XATTR_SHARED) +#ifndef S_IXUGO +#define S_IXUGO (S_IXUSR|S_IXGRP|S_IXOTH) +#endif + using namespace TOPNSPC::common; namespace bs = boost::system; @@ -156,6 +160,14 @@ void client_flush_set_callback(void *p, ObjectCacher::ObjectSet *oset) client->flush_set_callback(oset); } +bool Client::is_reserved_vino(vinodeno_t &vino) { + if (MDS_IS_PRIVATE_INO(vino.ino)) { + ldout(cct, -1) << __func__ << " attempt to access reserved inode number " << vino << dendl; + return true; + } + return false; +} + // ------------- @@ -196,6 +208,21 @@ int Client::CommandHook::call( // ------------- +int Client::get_fd_inode(int fd, InodeRef *in) { + int r = 0; + if (fd == CEPHFS_AT_FDCWD) { + *in = cwd; + } else { + Fh *f = get_filehandle(fd); + if (!f) { + r = -CEPHFS_EBADF; + } else { + *in = f->inode; + } + } + return r; +} + dir_result_t::dir_result_t(Inode *in, const UserPerm& perms) : inode(in), offset(0), next_offset(2), release_count(0), ordered_count(0), cache_index(0), start_shared_gen(0), @@ -391,13 +418,7 @@ void Client::tear_down_cache() // close root ino ceph_assert(inode_map.size() <= 1 + root_parents.size()); if (root && inode_map.size() == 1 + root_parents.size()) { - delete root; - root = 0; - root_ancestor = 0; - while (!root_parents.empty()) - root_parents.erase(root_parents.begin()); - inode_map.clear(); - _reset_faked_inos(); + root.reset(); } ceph_assert(inode_map.empty()); @@ -416,7 +437,7 @@ Inode *Client::get_root() { std::scoped_lock l(client_lock); root->ll_get(); - return root; + return root.get(); } @@ -430,7 +451,7 @@ void Client::dump_inode(Formatter *f, Inode *in, set& did, bool disconne << (disconnected ? "DISCONNECTED ":"") << "inode " << in->ino << " " << path - << " ref " << in->get_num_ref() + << " ref " << in->get_nref() << " " << *in << dendl; if (f) { @@ -470,7 +491,7 @@ void Client::dump_cache(Formatter *f) f->open_array_section("cache"); if (root) - dump_inode(f, root, did, true); + dump_inode(f, root.get(), did, true); // make a second pass to catch anything disconnected for (ceph::unordered_map::iterator it = inode_map.begin(); @@ -705,15 +726,9 @@ void Client::trim_cache(bool trim_kernel_dcache) _invalidate_kernel_dcache(); // hose root? - if (lru.lru_get_size() == 0 && root && root->get_num_ref() == 0 && inode_map.size() == 1 + root_parents.size()) { + if (lru.lru_get_size() == 0 && root && root->get_nref() == 1 && inode_map.size() == 1 + root_parents.size()) { ldout(cct, 15) << "trim_cache trimmed root " << root << dendl; - delete root; - root = 0; - root_ancestor = 0; - while (!root_parents.empty()) - root_parents.erase(root_parents.begin()); - inode_map.clear(); - _reset_faked_inos(); + root.reset(); } } @@ -892,7 +907,7 @@ Inode * Client::add_update_inode(InodeStat *st, utime_t from, if (!root) { root = in; if (use_faked_inos()) - _assign_faked_root(root); + _assign_faked_root(root.get()); root_ancestor = in; cwd = root; } else if (is_mounting()) { @@ -3168,8 +3183,11 @@ void Client::_put_inode(Inode *in, int n) { ldout(cct, 10) << __func__ << " on " << *in << " n = " << n << dendl; - int left = in->_put(n); - if (left == 0) { + int left = in->get_nref(); + ceph_assert(left >= n + 1); + in->iput(n); + left -= n; + if (left == 1) { // the last one will be held by the inode_map // release any caps remove_all_caps(in); @@ -3180,14 +3198,13 @@ void Client::_put_inode(Inode *in, int n) if (use_faked_inos()) _release_faked_ino(in); - if (in == root) { - root = 0; + if (root == nullptr) { root_ancestor = 0; while (!root_parents.empty()) root_parents.erase(root_parents.begin()); } - delete in; + in->iput(); } } @@ -3338,12 +3355,12 @@ void Client::get_cap_ref(Inode *in, int cap) if ((cap & CEPH_CAP_FILE_BUFFER) && in->cap_refs[CEPH_CAP_FILE_BUFFER] == 0) { ldout(cct, 5) << __func__ << " got first FILE_BUFFER ref on " << *in << dendl; - in->get(); + in->iget(); } if ((cap & CEPH_CAP_FILE_CACHE) && in->cap_refs[CEPH_CAP_FILE_CACHE] == 0) { ldout(cct, 5) << __func__ << " got first FILE_CACHE ref on " << *in << dendl; - in->get(); + in->iget(); } in->get_cap_ref(cap); } @@ -4538,7 +4555,7 @@ void Client::trim_caps(MetaSession *s, uint64_t max) ++q; if (dn->lru_is_expireable()) { if (can_invalidate_dentries && - dn->dir->parent_inode->ino == MDS_INO_ROOT) { + dn->dir->parent_inode->ino == CEPH_INO_ROOT) { // Only issue one of these per DN for inodes in root: handle // others more efficiently by calling for root-child DNs at // the end of this function. @@ -4551,10 +4568,10 @@ void Client::trim_caps(MetaSession *s, uint64_t max) all = false; } } - if (in->ll_ref == 1 && in->ino != MDS_INO_ROOT) { + if (in->ll_ref == 1 && in->ino != CEPH_INO_ROOT) { _schedule_ino_release_callback(in.get()); } - if (all && in->ino != MDS_INO_ROOT) { + if (all && in->ino != CEPH_INO_ROOT) { ldout(cct, 20) << __func__ << " counting as trimmed: " << *in << dendl; trimmed++; } @@ -5386,7 +5403,7 @@ void Client::_schedule_invalidate_dentry_callback(Dentry *dn, bool del) void Client::_try_to_trim_inode(Inode *in, bool sched_inval) { - int ref = in->get_num_ref(); + int ref = in->get_nref(); ldout(cct, 5) << __func__ << " in " << *in <dir && !in->dir->dentries.empty()) { @@ -5410,13 +5427,13 @@ void Client::_try_to_trim_inode(Inode *in, bool sched_inval) } } - if (ref > 0 && (in->flags & I_SNAPDIR_OPEN)) { + if (ref > 1 && (in->flags & I_SNAPDIR_OPEN)) { InodeRef snapdir = open_snapdir(in); _try_to_trim_inode(snapdir.get(), false); --ref; } - if (ref > 0) { + if (ref > 1) { auto q = in->dentries.begin(); while (q != in->dentries.end()) { Dentry *dn = *q; @@ -5589,8 +5606,12 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, const M int Client::inode_permission(Inode *in, const UserPerm& perms, unsigned want) { - if (perms.uid() == 0) + if (perms.uid() == 0) { + // Executable are overridable when there is at least one exec bit set + if((want & MAY_EXEC) && !(in->mode & S_IXUGO)) + return -CEPHFS_EACCES; return 0; + } if (perms.uid() != in->uid && (in->mode & S_IRWXG)) { int ret = _posix_acl_permission(in, perms, want); @@ -6224,7 +6245,7 @@ int Client::mount(const std::string &mount_root, const UserPerm& perms, } ceph_assert(root); - _ll_get(root); + _ll_get(root.get()); // trace? if (!cct->_conf->client_trace.empty()) { @@ -6385,6 +6406,7 @@ void Client::_unmount(bool abort) }); cwd.reset(); + root.reset(); // clean up any unclosed files while (!fd_map.empty()) { @@ -6917,25 +6939,30 @@ int Client::walk(std::string_view path, walk_dentry_result* wdr, const UserPerm& } int Client::path_walk(const filepath& origpath, InodeRef *end, - const UserPerm& perms, bool followsym, int mask) + const UserPerm& perms, bool followsym, int mask, InodeRef dirinode) { walk_dentry_result wdr; - int rc = path_walk(origpath, &wdr, perms, followsym, mask); + int rc = path_walk(origpath, &wdr, perms, followsym, mask, dirinode); *end = std::move(wdr.in); return rc; } -int Client::path_walk(const filepath& origpath, walk_dentry_result* result, const UserPerm& perms, bool followsym, int mask) +int Client::path_walk(const filepath& origpath, walk_dentry_result* result, const UserPerm& perms, + bool followsym, int mask, InodeRef dirinode) { filepath path = origpath; InodeRef cur; std::string alternate_name; if (origpath.absolute()) cur = root; - else + else if (!dirinode) cur = cwd; + else { + cur = dirinode; + } ceph_assert(cur); + ldout(cct, 20) << __func__ << " cur=" << *cur << dendl; ldout(cct, 10) << __func__ << " " << path << dendl; int symlinks = 0; @@ -7058,16 +7085,25 @@ int Client::link(const char *relexisting, const char *relpath, const UserPerm& p } int Client::unlink(const char *relpath, const UserPerm& perm) +{ + return unlinkat(CEPHFS_AT_FDCWD, relpath, 0, perm); +} + +int Client::unlinkat(int dirfd, const char *relpath, int flags, const UserPerm& perm) { RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); - if (!mref_reader.is_state_satisfied()) + if (!mref_reader.is_state_satisfied()) { return -CEPHFS_ENOTCONN; + } tout(cct) << __func__ << std::endl; + tout(cct) << dirfd << std::endl; tout(cct) << relpath << std::endl; + tout(cct) << flags << std::endl; - if (std::string(relpath) == "/") - return -CEPHFS_EISDIR; + if (std::string(relpath) == "/") { + return flags & AT_REMOVEDIR ? -CEPHFS_EBUSY : -CEPHFS_EISDIR; + } filepath path(relpath); string name = path.last_dentry(); @@ -7075,15 +7111,29 @@ int Client::unlink(const char *relpath, const UserPerm& perm) InodeRef dir; std::scoped_lock lock(client_lock); - int r = path_walk(path, &dir, perm); - if (r < 0) + + InodeRef dirinode; + int r = get_fd_inode(dirfd, &dirinode); + if (r < 0) { + return r; + } + + r = path_walk(path, &dir, perm, true, 0, dirinode); + if (r < 0) { return r; + } if (cct->_conf->client_permissions) { r = may_delete(dir.get(), name.c_str(), perm); - if (r < 0) + if (r < 0) { return r; + } } - return _unlink(dir.get(), name.c_str(), perm); + if (flags & AT_REMOVEDIR) { + r = _rmdir(dir.get(), name.c_str(), perm); + } else { + r = _unlink(dir.get(), name.c_str(), perm); + } + return r; } int Client::rename(const char *relfrom, const char *relto, const UserPerm& perm, std::string alternate_name) @@ -7132,18 +7182,26 @@ out: // dirs int Client::mkdir(const char *relpath, mode_t mode, const UserPerm& perm, std::string alternate_name) +{ + return mkdirat(CEPHFS_AT_FDCWD, relpath, mode, perm, alternate_name); +} + +int Client::mkdirat(int dirfd, const char *relpath, mode_t mode, const UserPerm& perm, + std::string alternate_name) { RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); if (!mref_reader.is_state_satisfied()) return -CEPHFS_ENOTCONN; tout(cct) << __func__ << std::endl; + tout(cct) << dirfd << std::endl; tout(cct) << relpath << std::endl; tout(cct) << mode << std::endl; ldout(cct, 10) << __func__ << ": " << relpath << dendl; - if (std::string(relpath) == "/") + if (std::string(relpath) == "/") { return -CEPHFS_EEXIST; + } filepath path(relpath); string name = path.last_dentry(); @@ -7151,13 +7209,22 @@ int Client::mkdir(const char *relpath, mode_t mode, const UserPerm& perm, std::s InodeRef dir; std::scoped_lock lock(client_lock); - int r = path_walk(path, &dir, perm); - if (r < 0) + + InodeRef dirinode; + int r = get_fd_inode(dirfd, &dirinode); + if (r < 0) { return r; + } + + r = path_walk(path, &dir, perm, true, 0, dirinode); + if (r < 0) { + return r; + } if (cct->_conf->client_permissions) { r = may_create(dir.get(), perm); - if (r < 0) + if (r < 0) { return r; + } } return _mkdir(dir.get(), name.c_str(), mode, perm, 0, {}, std::move(alternate_name)); } @@ -7221,31 +7288,7 @@ int Client::mkdirs(const char *relpath, mode_t mode, const UserPerm& perms) int Client::rmdir(const char *relpath, const UserPerm& perms) { - RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); - if (!mref_reader.is_state_satisfied()) - return -CEPHFS_ENOTCONN; - - tout(cct) << __func__ << std::endl; - tout(cct) << relpath << std::endl; - - if (std::string(relpath) == "/") - return -CEPHFS_EBUSY; - - filepath path(relpath); - string name = path.last_dentry(); - path.pop_dentry(); - InodeRef dir; - - std::scoped_lock lock(client_lock); - int r = path_walk(path, &dir, perms); - if (r < 0) - return r; - if (cct->_conf->client_permissions) { - int r = may_delete(dir.get(), name.c_str(), perms); - if (r < 0) - return r; - } - return _rmdir(dir.get(), name.c_str(), perms); + return unlinkat(CEPHFS_AT_FDCWD, relpath, AT_REMOVEDIR, perms); } int Client::mknod(const char *relpath, mode_t mode, const UserPerm& perms, dev_t rdev) @@ -7282,17 +7325,26 @@ int Client::mknod(const char *relpath, mode_t mode, const UserPerm& perms, dev_t // symlinks int Client::symlink(const char *target, const char *relpath, const UserPerm& perms, std::string alternate_name) +{ + return symlinkat(target, CEPHFS_AT_FDCWD, relpath, perms, alternate_name); +} + +int Client::symlinkat(const char *target, int dirfd, const char *relpath, const UserPerm& perms, + std::string alternate_name) { RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); - if (!mref_reader.is_state_satisfied()) + if (!mref_reader.is_state_satisfied()) { return -CEPHFS_ENOTCONN; + } tout(cct) << __func__ << std::endl; tout(cct) << target << std::endl; + tout(cct) << dirfd << std::endl; tout(cct) << relpath << std::endl; - if (std::string(relpath) == "/") + if (std::string(relpath) == "/") { return -CEPHFS_EEXIST; + } filepath path(relpath); string name = path.last_dentry(); @@ -7300,33 +7352,53 @@ int Client::symlink(const char *target, const char *relpath, const UserPerm& per InodeRef dir; std::scoped_lock lock(client_lock); - int r = path_walk(path, &dir, perms); - if (r < 0) + + InodeRef dirinode; + int r = get_fd_inode(dirfd, &dirinode); + if (r < 0) { return r; + } + r = path_walk(path, &dir, perms, true, 0, dirinode); + if (r < 0) { + return r; + } if (cct->_conf->client_permissions) { int r = may_create(dir.get(), perms); - if (r < 0) + if (r < 0) { return r; + } } return _symlink(dir.get(), name.c_str(), target, perms, std::move(alternate_name)); } int Client::readlink(const char *relpath, char *buf, loff_t size, const UserPerm& perms) { + return readlinkat(CEPHFS_AT_FDCWD, relpath, buf, size, perms); +} + +int Client::readlinkat(int dirfd, const char *relpath, char *buf, loff_t size, const UserPerm& perms) { RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); - if (!mref_reader.is_state_satisfied()) + if (!mref_reader.is_state_satisfied()) { return -CEPHFS_ENOTCONN; + } tout(cct) << __func__ << std::endl; + tout(cct) << dirfd << std::endl; tout(cct) << relpath << std::endl; - filepath path(relpath); - InodeRef in; - + InodeRef dirinode; std::scoped_lock lock(client_lock); - int r = path_walk(path, &in, perms, false); - if (r < 0) + int r = get_fd_inode(dirfd, &dirinode); + if (r < 0) { return r; + } + + InodeRef in; + filepath path(relpath); + r = path_walk(path, &in, perms, false, 0, dirinode); + if (r < 0) { + return r; + } return _readlink(in.get(), buf, size); } @@ -7759,33 +7831,7 @@ int Client::statx(const char *relpath, struct ceph_statx *stx, const UserPerm& perms, unsigned int want, unsigned int flags) { - RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); - if (!mref_reader.is_state_satisfied()) - return -CEPHFS_ENOTCONN; - - ldout(cct, 3) << __func__ << " enter (relpath " << relpath << " want " << want << ")" << dendl; - tout(cct) << "statx" << std::endl; - tout(cct) << relpath << std::endl; - - filepath path(relpath); - InodeRef in; - - unsigned mask = statx_to_mask(flags, want); - - std::scoped_lock lock(client_lock); - int r = path_walk(path, &in, perms, !(flags & AT_SYMLINK_NOFOLLOW), mask); - if (r < 0) - return r; - - r = _getattr(in, mask, perms); - if (r < 0) { - ldout(cct, 3) << __func__ << " exit on error!" << dendl; - return r; - } - - fill_statx(in, mask, stx); - ldout(cct, 3) << __func__ << " exit (relpath " << relpath << " mask " << stx->stx_mask << ")" << dendl; - return r; + return statxat(CEPHFS_AT_FDCWD, relpath, stx, perms, want, flags); } int Client::lstat(const char *relpath, struct stat *stbuf, @@ -7977,24 +8023,7 @@ void Client::touch_dn(Dentry *dn) int Client::chmod(const char *relpath, mode_t mode, const UserPerm& perms) { - RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); - if (!mref_reader.is_state_satisfied()) - return -CEPHFS_ENOTCONN; - - tout(cct) << __func__ << std::endl; - tout(cct) << relpath << std::endl; - tout(cct) << mode << std::endl; - - filepath path(relpath); - InodeRef in; - - std::scoped_lock lock(client_lock); - int r = path_walk(path, &in, perms); - if (r < 0) - return r; - struct stat attr; - attr.st_mode = mode; - return _setattr(in, &attr, CEPH_SETATTR_MODE, perms); + return chmodat(CEPHFS_AT_FDCWD, relpath, mode, 0, perms); } int Client::fchmod(int fd, mode_t mode, const UserPerm& perms) @@ -8020,52 +8049,47 @@ int Client::fchmod(int fd, mode_t mode, const UserPerm& perms) return _setattr(f->inode, &attr, CEPH_SETATTR_MODE, perms); } -int Client::lchmod(const char *relpath, mode_t mode, const UserPerm& perms) -{ +int Client::chmodat(int dirfd, const char *relpath, mode_t mode, int flags, + const UserPerm& perms) { RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); - if (!mref_reader.is_state_satisfied()) + if (!mref_reader.is_state_satisfied()) { return -CEPHFS_ENOTCONN; + } tout(cct) << __func__ << std::endl; + tout(cct) << dirfd << std::endl; tout(cct) << relpath << std::endl; tout(cct) << mode << std::endl; + tout(cct) << flags << std::endl; filepath path(relpath); InodeRef in; + InodeRef dirinode; std::scoped_lock lock(client_lock); - // don't follow symlinks - int r = path_walk(path, &in, perms, false); - if (r < 0) + int r = get_fd_inode(dirfd, &dirinode); + if (r < 0) { + return r; + } + + r = path_walk(path, &in, perms, !(flags & AT_SYMLINK_NOFOLLOW), 0, dirinode); + if (r < 0) { return r; + } struct stat attr; attr.st_mode = mode; return _setattr(in, &attr, CEPH_SETATTR_MODE, perms); } +int Client::lchmod(const char *relpath, mode_t mode, const UserPerm& perms) +{ + return chmodat(CEPHFS_AT_FDCWD, relpath, mode, AT_SYMLINK_NOFOLLOW, perms); +} + int Client::chown(const char *relpath, uid_t new_uid, gid_t new_gid, const UserPerm& perms) { - RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); - if (!mref_reader.is_state_satisfied()) - return -CEPHFS_ENOTCONN; - - tout(cct) << __func__ << std::endl; - tout(cct) << relpath << std::endl; - tout(cct) << new_uid << std::endl; - tout(cct) << new_gid << std::endl; - - filepath path(relpath); - InodeRef in; - - std::scoped_lock lock(client_lock); - int r = path_walk(path, &in, perms); - if (r < 0) - return r; - struct stat attr; - attr.st_uid = new_uid; - attr.st_gid = new_gid; - return _setattr(in, &attr, CEPH_SETATTR_UID|CEPH_SETATTR_GID, perms); + return chownat(CEPHFS_AT_FDCWD, relpath, new_uid, new_gid, 0, perms); } int Client::fchown(int fd, uid_t new_uid, gid_t new_gid, const UserPerm& perms) @@ -8099,30 +8123,41 @@ int Client::fchown(int fd, uid_t new_uid, gid_t new_gid, const UserPerm& perms) int Client::lchown(const char *relpath, uid_t new_uid, gid_t new_gid, const UserPerm& perms) { + return chownat(CEPHFS_AT_FDCWD, relpath, new_uid, new_gid, AT_SYMLINK_NOFOLLOW, perms); +} + +int Client::chownat(int dirfd, const char *relpath, uid_t new_uid, gid_t new_gid, + int flags, const UserPerm& perms) { RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); - if (!mref_reader.is_state_satisfied()) + if (!mref_reader.is_state_satisfied()) { return -CEPHFS_ENOTCONN; + } tout(cct) << __func__ << std::endl; + tout(cct) << dirfd << std::endl; tout(cct) << relpath << std::endl; tout(cct) << new_uid << std::endl; tout(cct) << new_gid << std::endl; + tout(cct) << flags << std::endl; filepath path(relpath); InodeRef in; + InodeRef dirinode; std::scoped_lock lock(client_lock); - // don't follow symlinks - int r = path_walk(path, &in, perms, false); - if (r < 0) + int r = get_fd_inode(dirfd, &dirinode); + if (r < 0) { return r; + } + + r = path_walk(path, &in, perms, !(flags & AT_SYMLINK_NOFOLLOW), 0, dirinode); + if (r < 0) { + return r; + } struct stat attr; attr.st_uid = new_uid; attr.st_gid = new_gid; - int mask = 0; - if (new_uid != static_cast(-1)) mask |= CEPH_SETATTR_UID; - if (new_gid != static_cast(-1)) mask |= CEPH_SETATTR_GID; - return _setattr(in, &attr, mask, perms); + return _setattr(in, &attr, CEPH_SETATTR_UID|CEPH_SETATTR_GID, perms); } static void attr_set_atime_and_mtime(struct stat *attr, @@ -8271,6 +8306,50 @@ int Client::futimens(int fd, struct timespec times[2], const UserPerm& perms) return _setattr(f->inode, &attr, CEPH_SETATTR_MTIME|CEPH_SETATTR_ATIME, perms); } +int Client::utimensat(int dirfd, const char *relpath, struct timespec times[2], int flags, + const UserPerm& perms) { + RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); + if (!mref_reader.is_state_satisfied()) { + return -CEPHFS_ENOTCONN; + } + + tout(cct) << __func__ << std::endl; + tout(cct) << dirfd << std::endl; + tout(cct) << relpath << std::endl; + tout(cct) << "atime: " << times[0].tv_sec << "." << times[0].tv_nsec + << std::endl; + tout(cct) << "mtime: " << times[1].tv_sec << "." << times[1].tv_nsec + << std::endl; + tout(cct) << flags << std::endl; + + filepath path(relpath); + InodeRef in; + InodeRef dirinode; + + std::scoped_lock lock(client_lock); + int r = get_fd_inode(dirfd, &dirinode); + if (r < 0) { + return r; + } + +#if defined(__linux__) && defined(O_PATH) + if (flags & O_PATH) { + return -CEPHFS_EBADF; + } +#endif + + r = path_walk(path, &in, perms, !(flags & AT_SYMLINK_NOFOLLOW), 0, dirinode); + if (r < 0) { + return r; + } + struct stat attr; + utime_t atime(times[0]); + utime_t mtime(times[1]); + + attr_set_atime_and_mtime(&attr, atime, mtime); + return _setattr(in, &attr, CEPH_SETATTR_MTIME|CEPH_SETATTR_ATIME, perms); +} + int Client::flock(int fd, int operation, uint64_t owner) { RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); @@ -8318,6 +8397,36 @@ int Client::opendir(const char *relpath, dir_result_t **dirpp, const UserPerm& p return r; } +int Client::fdopendir(int dirfd, dir_result_t **dirpp, const UserPerm &perms) { + RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); + if (!mref_reader.is_state_satisfied()) { + return -CEPHFS_ENOTCONN; + } + + tout(cct) << __func__ << std::endl; + tout(cct) << dirfd << std::endl; + + InodeRef dirinode; + std::scoped_lock locker(client_lock); + int r = get_fd_inode(dirfd, &dirinode); + if (r < 0) { + return r; + } + + if (cct->_conf->client_permissions) { + r = may_open(dirinode.get(), O_RDONLY, perms); + if (r < 0) { + return r; + } + } + r = _opendir(dirinode.get(), dirpp, perms); + /* if ENOTDIR, dirpp will be an uninitialized point and it's very dangerous to access its value */ + if (r != -CEPHFS_ENOTDIR) { + tout(cct) << (uintptr_t)*dirpp << std::endl; + } + return r; +} + int Client::_opendir(Inode *in, dir_result_t **dirpp, const UserPerm& perms) { if (!in->is_dir()) @@ -9016,19 +9125,14 @@ int Client::getdir(const char *relpath, list& contents, /****** file i/o **********/ -int Client::open(const char *relpath, int flags, const UserPerm& perms, - mode_t mode, int stripe_unit, int stripe_count, - int object_size, const char *data_pool, std::string alternate_name) -{ - RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); - if (!mref_reader.is_state_satisfied()) - return -CEPHFS_ENOTCONN; +// common parts for open and openat. call with client_lock locked. +int Client::create_and_open(std::optional dirfd, const char *relpath, int flags, + const UserPerm& perms, mode_t mode, int stripe_unit, + int stripe_count, int object_size, const char *data_pool, + std::string alternate_name) { + ceph_assert(ceph_mutex_is_locked(client_lock)); int cflags = ceph_flags_sys2wire(flags); - - ldout(cct, 3) << "open enter(" << relpath << ", " << cflags << "," << mode << ")" << dendl; - tout(cct) << "open" << std::endl; - tout(cct) << relpath << std::endl; tout(cct) << cflags << std::endl; Fh *fh = NULL; @@ -9048,16 +9152,22 @@ int Client::open(const char *relpath, int flags, const UserPerm& perms, bool followsym = !((flags & O_NOFOLLOW) || ((flags & O_CREAT) && (flags & O_EXCL))); int mask = ceph_caps_for_mode(ceph_flags_to_mode(cflags)); - std::scoped_lock lock(client_lock); - int r = path_walk(path, &in, perms, followsym, mask); + InodeRef dirinode = nullptr; + if (dirfd) { + int r = get_fd_inode(*dirfd, &dirinode); + if (r < 0) { + return r; + } + } + int r = path_walk(path, &in, perms, followsym, mask, dirinode); if (r == 0 && (flags & O_CREAT) && (flags & O_EXCL)) return -CEPHFS_EEXIST; #if defined(__linux__) && defined(O_PATH) if (r == 0 && in->is_symlink() && (flags & O_NOFOLLOW) && !(flags & O_PATH)) #else - if (r == 0 && in->is_symlink() && (flags & O_NOFOLLOW)) + if (r == 0 && in->is_symlink() && (flags & O_NOFOLLOW)) #endif return -CEPHFS_ELOOP; @@ -9067,13 +9177,14 @@ int Client::open(const char *relpath, int flags, const UserPerm& perms, dirpath.pop_dentry(); InodeRef dir; r = path_walk(dirpath, &dir, perms, true, - cct->_conf->client_permissions ? CEPH_CAP_AUTH_SHARED : 0); - if (r < 0) + cct->_conf->client_permissions ? CEPH_CAP_AUTH_SHARED : 0, dirinode); + if (r < 0) { goto out; + } if (cct->_conf->client_permissions) { r = may_create(dir.get(), perms); if (r < 0) - goto out; + goto out; } r = _create(dir.get(), dname.c_str(), flags, mode, &in, &fh, stripe_unit, stripe_count, object_size, data_pool, &created, perms, @@ -9087,7 +9198,7 @@ int Client::open(const char *relpath, int flags, const UserPerm& perms, if (cct->_conf->client_permissions) { r = may_open(in.get(), flags, perms); if (r < 0) - goto out; + goto out; } } @@ -9102,8 +9213,42 @@ int Client::open(const char *relpath, int flags, const UserPerm& perms, } out: + return r; +} + +int Client::open(const char *relpath, int flags, const UserPerm& perms, + mode_t mode, int stripe_unit, int stripe_count, + int object_size, const char *data_pool, std::string alternate_name) +{ + return openat(CEPHFS_AT_FDCWD, relpath, flags, perms, mode, stripe_unit, + stripe_count, object_size, data_pool, alternate_name); +} + +int Client::_openat(int dirfd, const char *relpath, int flags, const UserPerm& perms, + mode_t mode, std::string alternate_name) { + return create_and_open(dirfd, relpath, flags, perms, mode, 0, 0, 0, NULL, alternate_name); +} + +int Client::openat(int dirfd, const char *relpath, int flags, const UserPerm& perms, + mode_t mode, int stripe_unit, int stripe_count, int object_size, + const char *data_pool, std::string alternate_name) { + RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); + if (!mref_reader.is_state_satisfied()) { + return -CEPHFS_ENOTCONN; + } + + ldout(cct, 3) << "openat enter(" << relpath << ")" << dendl; + tout(cct) << dirfd << std::endl; + tout(cct) << relpath << std::endl; + tout(cct) << flags << std::endl; + tout(cct) << mode << std::endl; + + std::scoped_lock locker(client_lock); + int r = create_and_open(dirfd, relpath, flags, perms, mode, stripe_unit, stripe_count, + object_size, data_pool, alternate_name); + tout(cct) << r << std::endl; - ldout(cct, 3) << "open exit(" << path << ", " << cflags << ") = " << r << dendl; + ldout(cct, 3) << "openat exit(" << relpath << ")" << dendl; return r; } @@ -9150,6 +9295,9 @@ int Client::_lookup_vino(vinodeno_t vino, const UserPerm& perms, Inode **inode) if (!mref_reader.is_state_satisfied()) return -CEPHFS_ENOTCONN; + if (is_reserved_vino(vino)) + return -CEPHFS_ESTALE; + MetaRequest *req = new MetaRequest(CEPH_MDS_OP_LOOKUPINO); filepath path(vino.ino); req->set_filepath(path); @@ -9432,17 +9580,12 @@ int Client::_renew_caps(Inode *in) return ret; } -int Client::close(int fd) +int Client::_close(int fd) { - RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); - if (!mref_reader.is_state_satisfied()) - return -CEPHFS_ENOTCONN; - ldout(cct, 3) << "close enter(" << fd << ")" << dendl; tout(cct) << "close" << std::endl; tout(cct) << fd << std::endl; - std::scoped_lock lock(client_lock); Fh *fh = get_filehandle(fd); if (!fh) return -CEPHFS_EBADF; @@ -9453,6 +9596,14 @@ int Client::close(int fd) return err; } +int Client::close(int fd) { + RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); + if (!mref_reader.is_state_satisfied()) + return -CEPHFS_ENOTCONN; + + std::scoped_lock lock(client_lock); + return _close(fd); +} // ------------ // read, write @@ -10501,7 +10652,7 @@ int Client::fstatx(int fd, struct ceph_statx *stx, const UserPerm& perms, unsigned mask = statx_to_mask(flags, want); int r = 0; - if (mask && !f->inode->caps_issued_mask(mask, true)) { + if (mask) { r = _getattr(f->inode, mask, perms); if (r < 0) { ldout(cct, 3) << "fstatx exit on error!" << dendl; @@ -10514,6 +10665,44 @@ int Client::fstatx(int fd, struct ceph_statx *stx, const UserPerm& perms, return r; } +int Client::statxat(int dirfd, const char *relpath, + struct ceph_statx *stx, const UserPerm& perms, + unsigned int want, unsigned int flags) { + RWRef_t mref_reader(mount_state, CLIENT_MOUNTING); + if (!mref_reader.is_state_satisfied()) { + return -CEPHFS_ENOTCONN; + } + + tout(cct) << __func__ << " flags " << hex << flags << " want " << want << dec << std::endl; + tout(cct) << dirfd << std::endl; + tout(cct) << relpath << std::endl; + + unsigned mask = statx_to_mask(flags, want); + + InodeRef dirinode; + std::scoped_lock lock(client_lock); + int r = get_fd_inode(dirfd, &dirinode); + if (r < 0) { + return r; + } + + InodeRef in; + filepath path(relpath); + r = path_walk(path, &in, perms, !(flags & AT_SYMLINK_NOFOLLOW), mask, dirinode); + if (r < 0) { + return r; + } + r = _getattr(in, mask, perms); + if (r < 0) { + ldout(cct, 3) << __func__ << " exit on error!" << dendl; + return r; + } + + fill_statx(in, mask, stx); + ldout(cct, 3) << __func__ << " dirfd" << dirfd << ", r= " << r << dendl; + return r; +} + // not written yet, but i want to link! int Client::chdir(const char *relpath, std::string &new_cwd, @@ -10551,7 +10740,7 @@ void Client::_getcwd(string& dir, const UserPerm& perms) ldout(cct, 10) << __func__ << " " << *cwd << dendl; Inode *in = cwd.get(); - while (in != root) { + while (in != root.get()) { ceph_assert(in->dentries.size() < 2); // dirs can't be hard-linked // A cwd or ancester is unlinked @@ -10654,7 +10843,7 @@ int Client::statfs(const char *path, struct statvfs *stbuf, // quota but we can see a parent of it that does have a quota, we'll // respect that one instead. ceph_assert(root != nullptr); - Inode *quota_root = root->quota.is_enable() ? root : get_quota_root(root, perms); + InodeRef quota_root = root->quota.is_enable() ? root : get_quota_root(root.get(), perms); // get_quota_root should always give us something // because client quotas are always enabled @@ -11406,6 +11595,9 @@ int Client::ll_lookup_vino( if (!mref_reader.is_state_satisfied()) return -CEPHFS_ENOTCONN; + if (is_reserved_vino(vino)) + return -CEPHFS_ESTALE; + std::scoped_lock lock(client_lock); ldout(cct, 3) << __func__ << " " << vino << dendl; @@ -11527,7 +11719,7 @@ int Client::ll_walk(const char* name, Inode **out, struct ceph_statx *stx, void Client::_ll_get(Inode *in) { if (in->ll_ref == 0) { - in->get(); + in->iget(); if (in->is_dir() && !in->dentries.empty()) { ceph_assert(in->dentries.size() == 1); // dirs can't be hard-linked in->get_first_parent()->get(); // pin dentry @@ -11660,6 +11852,9 @@ Inode *Client::ll_get_inode(vinodeno_t vino) if (!mref_reader.is_state_satisfied()) return NULL; + if (is_reserved_vino(vino)) + return NULL; + std::scoped_lock lock(client_lock); unordered_map::iterator p = inode_map.find(vino); @@ -14776,7 +14971,7 @@ void Client::ms_handle_remote_reset(Connection *con) mds_rank_t mds = MDS_RANK_NONE; MetaSession *s = NULL; for (auto &p : mds_sessions) { - if (mdsmap->get_addrs(p.first) == con->get_peer_addrs()) { + if (mdsmap->have_inst(p.first) && mdsmap->get_addrs(p.first) == con->get_peer_addrs()) { mds = p.first; s = &p.second; } @@ -15364,7 +15559,7 @@ void Client::handle_conf_change(const ConfigProxy& conf, void intrusive_ptr_add_ref(Inode *in) { - in->get(); + in->iget(); } void intrusive_ptr_release(Inode *in) diff --git a/ceph/src/client/Client.h b/ceph/src/client/Client.h index 011ff1ad0..88e8f4429 100644 --- a/ceph/src/client/Client.h +++ b/ceph/src/client/Client.h @@ -331,6 +331,7 @@ public: // namespace ops int opendir(const char *name, dir_result_t **dirpp, const UserPerm& perms); + int fdopendir(int dirfd, dir_result_t **dirpp, const UserPerm& perms); int closedir(dir_result_t *dirp); /** @@ -372,17 +373,23 @@ public: int may_delete(const char *relpath, const UserPerm& perms); int link(const char *existing, const char *newname, const UserPerm& perm, std::string alternate_name=""); int unlink(const char *path, const UserPerm& perm); + int unlinkat(int dirfd, const char *relpath, int flags, const UserPerm& perm); int rename(const char *from, const char *to, const UserPerm& perm, std::string alternate_name=""); // dirs int mkdir(const char *path, mode_t mode, const UserPerm& perm, std::string alternate_name=""); + int mkdirat(int dirfd, const char *relpath, mode_t mode, const UserPerm& perm, + std::string alternate_name=""); int mkdirs(const char *path, mode_t mode, const UserPerm& perms); int rmdir(const char *path, const UserPerm& perms); // symlinks int readlink(const char *path, char *buf, loff_t size, const UserPerm& perms); + int readlinkat(int dirfd, const char *relpath, char *buf, loff_t size, const UserPerm& perms); int symlink(const char *existing, const char *newname, const UserPerm& perms, std::string alternate_name=""); + int symlinkat(const char *target, int dirfd, const char *relpath, const UserPerm& perms, + std::string alternate_name=""); // path traversal for high-level interface int walk(std::string_view path, struct walk_dentry_result* result, const UserPerm& perms, bool followsym=true); @@ -405,12 +412,15 @@ public: int fsetattrx(int fd, struct ceph_statx *stx, int mask, const UserPerm& perms); int chmod(const char *path, mode_t mode, const UserPerm& perms); int fchmod(int fd, mode_t mode, const UserPerm& perms); + int chmodat(int dirfd, const char *relpath, mode_t mode, int flags, const UserPerm& perms); int lchmod(const char *path, mode_t mode, const UserPerm& perms); int chown(const char *path, uid_t new_uid, gid_t new_gid, const UserPerm& perms); int fchown(int fd, uid_t new_uid, gid_t new_gid, const UserPerm& perms); int lchown(const char *path, uid_t new_uid, gid_t new_gid, const UserPerm& perms); + int chownat(int dirfd, const char *relpath, uid_t new_uid, gid_t new_gid, + int flags, const UserPerm& perms); int utime(const char *path, struct utimbuf *buf, const UserPerm& perms); int lutime(const char *path, struct utimbuf *buf, const UserPerm& perms); int futime(int fd, struct utimbuf *buf, const UserPerm& perms); @@ -418,21 +428,38 @@ public: int lutimes(const char *relpath, struct timeval times[2], const UserPerm& perms); int futimes(int fd, struct timeval times[2], const UserPerm& perms); int futimens(int fd, struct timespec times[2], const UserPerm& perms); + int utimensat(int dirfd, const char *relpath, struct timespec times[2], int flags, + const UserPerm& perms); int flock(int fd, int operation, uint64_t owner); int truncate(const char *path, loff_t size, const UserPerm& perms); // file ops int mknod(const char *path, mode_t mode, const UserPerm& perms, dev_t rdev=0); + + int create_and_open(std::optional dirfd, const char *relpath, int flags, const UserPerm& perms, + mode_t mode, int stripe_unit, int stripe_count, int object_size, const char *data_pool, + std::string alternate_name); int open(const char *path, int flags, const UserPerm& perms, mode_t mode=0, std::string alternate_name="") { return open(path, flags, perms, mode, 0, 0, 0, NULL, alternate_name); } int open(const char *path, int flags, const UserPerm& perms, mode_t mode, int stripe_unit, int stripe_count, int object_size, const char *data_pool, std::string alternate_name=""); + int _openat(int dirfd, const char *relpath, int flags, const UserPerm& perms, + mode_t mode=0, std::string alternate_name=""); + int openat(int dirfd, const char *relpath, int flags, const UserPerm& perms, + mode_t mode, int stripe_unit, int stripe_count, + int object_size, const char *data_pool, std::string alternate_name); + int openat(int dirfd, const char *path, int flags, const UserPerm& perms, mode_t mode=0, + std::string alternate_name="") { + return openat(dirfd, path, flags, perms, mode, 0, 0, 0, NULL, alternate_name); + } + int lookup_hash(inodeno_t ino, inodeno_t dirino, const char *name, const UserPerm& perms); int lookup_ino(inodeno_t ino, const UserPerm& perms, Inode **inode=NULL); int lookup_name(Inode *in, Inode *parent, const UserPerm& perms); + int _close(int fd); int close(int fd); loff_t lseek(int fd, loff_t offset, int whence); int read(int fd, char *buf, loff_t size, loff_t offset=-1); @@ -446,6 +473,9 @@ public: int mask=CEPH_STAT_CAP_INODE_ALL); int fstatx(int fd, struct ceph_statx *stx, const UserPerm& perms, unsigned int want, unsigned int flags); + int statxat(int dirfd, const char *relpath, + struct ceph_statx *stx, const UserPerm& perms, + unsigned int want, unsigned int flags); int fallocate(int fd, int mode, loff_t offset, loff_t length); // full path xattr ops @@ -909,9 +939,10 @@ protected: void handle_client_reply(const MConstRef& reply); bool is_dir_operation(MetaRequest *request); - int path_walk(const filepath& fp, struct walk_dentry_result* result, const UserPerm& perms, bool followsym=true, int mask=0); + int path_walk(const filepath& fp, struct walk_dentry_result* result, const UserPerm& perms, bool followsym=true, int mask=0, + InodeRef dirinode=nullptr); int path_walk(const filepath& fp, InodeRef *end, const UserPerm& perms, - bool followsym=true, int mask=0); + bool followsym=true, int mask=0, InodeRef dirinode=nullptr); // fake inode number for 32-bits ino_t void _assign_faked_ino(Inode *in); @@ -950,6 +981,7 @@ protected: return NULL; return it->second; } + int get_fd_inode(int fd, InodeRef *in); // helpers void wake_up_session_caps(MetaSession *s, bool reconnect); @@ -1056,7 +1088,7 @@ protected: std::map ll_snap_ref; - Inode* root = nullptr; + InodeRef root = nullptr; map root_parents; Inode* root_ancestor = nullptr; LRU lru; // lru list of Dentry's in our local metadata cache. @@ -1223,6 +1255,7 @@ private: static const VXattr _common_vxattrs[]; + bool is_reserved_vino(vinodeno_t &vino); void fill_dirent(struct dirent *de, const char *name, int type, uint64_t ino, loff_t next_off); diff --git a/ceph/src/client/Inode.cc b/ceph/src/client/Inode.cc index 2aab5ef30..efeedb1ef 100644 --- a/ceph/src/client/Inode.cc +++ b/ceph/src/client/Inode.cc @@ -40,7 +40,7 @@ ostream& operator<<(ostream &out, const Inode &in) { out << in.vino() << "(" << "faked_ino=" << in.faked_ino - << " ref=" << in._ref + << " nref=" << in.get_nref() << " ll_ref=" << in.ll_ref << " cap_refs=" << in.cap_refs << " open=" << in.open_by_mode @@ -138,8 +138,9 @@ void Inode::make_nosnap_relative_path(filepath& p) void Inode::get_open_ref(int mode) { client->inc_opened_files(); - if (open_by_mode.count(mode) == 0) + if (open_by_mode[mode] == 0) { client->inc_opened_inodes(); + } open_by_mode[mode]++; break_deleg(!(mode & CEPH_FILE_MODE_WR)); } @@ -214,8 +215,7 @@ int Inode::caps_issued(int *implemented) const { int c = snap_caps; int i = 0; - for (const auto &pair : caps) { - const Cap &cap = pair.second; + for (const auto &[mds, cap] : caps) { if (cap_is_valid(cap)) { c |= cap.issued; i |= cap.implemented; @@ -305,22 +305,18 @@ bool Inode::caps_issued_mask(unsigned mask, bool allow_impl) int Inode::caps_used() { int w = 0; - for (map::iterator p = cap_refs.begin(); - p != cap_refs.end(); - ++p) - if (p->second) - w |= p->first; + for (const auto &[cap, cnt] : cap_refs) + if (cnt) + w |= cap; return w; } int Inode::caps_file_wanted() { int want = 0; - for (map::iterator p = open_by_mode.begin(); - p != open_by_mode.end(); - ++p) - if (p->second) - want |= ceph_caps_for_mode(p->first); + for (const auto &[mode, cnt] : open_by_mode) + if (cnt) + want |= ceph_caps_for_mode(mode); return want; } @@ -384,7 +380,7 @@ Dir *Inode::open_dir() ceph_assert(dentries.size() < 2); // dirs can't be hard-linked if (!dentries.empty()) get_first_parent()->get(); // pin dentry - get(); // pin inode + iget(); // pin inode } return dir; } @@ -402,22 +398,6 @@ bool Inode::check_mode(const UserPerm& perms, unsigned want) return (mode & want) == want; } -void Inode::get() { - _ref++; - lsubdout(client->cct, client, 15) << "inode.get on " << this << " " << ino << '.' << snapid - << " now " << _ref << dendl; -} - -//private method to put a reference; see Client::put_inode() -int Inode::_put(int n) { - _ref -= n; - lsubdout(client->cct, client, 15) << "inode.put on " << this << " " << ino << '.' << snapid - << " now " << _ref << dendl; - ceph_assert(_ref >= 0); - return _ref; -} - - void Inode::dump(Formatter *f) const { f->dump_stream("ino") << ino; @@ -563,7 +543,7 @@ void Inode::dump(Formatter *f) const if (requested_max_size != max_size) f->dump_unsigned("requested_max_size", requested_max_size); - f->dump_int("ref", _ref); + f->dump_int("nref", get_nref()); f->dump_int("ll_ref", ll_ref); if (!dentries.empty()) { @@ -795,7 +775,7 @@ void Inode::mark_caps_dirty(int caps) lsubdout(client->cct, client, 10) << __func__ << " " << *this << " " << ccap_string(dirty_caps) << " -> " << ccap_string(dirty_caps | caps) << dendl; if (caps && !caps_dirty()) - get(); + iget(); dirty_caps |= caps; client->get_dirty_list().push_back(&dirty_cap_item); } diff --git a/ceph/src/client/Inode.h b/ceph/src/client/Inode.h index 4fa9c6938..86c08871a 100644 --- a/ceph/src/client/Inode.h +++ b/ceph/src/client/Inode.h @@ -78,32 +78,29 @@ struct CapSnap { //snapid_t follows; // map key InodeRef in; SnapContext context; - int issued, dirty; + int issued = 0, dirty = 0; - uint64_t size; + uint64_t size = 0; utime_t ctime, btime, mtime, atime; - version_t time_warp_seq; - uint64_t change_attr; - uint32_t mode; - uid_t uid; - gid_t gid; + version_t time_warp_seq = 0; + uint64_t change_attr = 0; + uint32_t mode = 0; + uid_t uid = 0; + gid_t gid = 0; map xattrs; - version_t xattr_version; + version_t xattr_version = 0; bufferlist inline_data; - version_t inline_version; + version_t inline_version = 0; - bool writing, dirty_data; - uint64_t flush_tid; + bool writing = false, dirty_data = false; + uint64_t flush_tid = 0; - int64_t cap_dirtier_uid; - int64_t cap_dirtier_gid; + int64_t cap_dirtier_uid = -1; + int64_t cap_dirtier_gid = -1; explicit CapSnap(Inode *i) - : in(i), issued(0), dirty(0), size(0), time_warp_seq(0), change_attr(0), - mode(0), uid(0), gid(0), xattr_version(0), inline_version(0), - writing(false), dirty_data(false), flush_tid(0), cap_dirtier_uid(-1), - cap_dirtier_gid(-1) + : in(i) {} void dump(Formatter *f) const; @@ -117,58 +114,58 @@ struct CapSnap { #define I_CAP_DROPPED (1 << 4) #define I_ERROR_FILELOCK (1 << 5) -struct Inode { +struct Inode : RefCountedObject { Client *client; // -- the actual inode -- inodeno_t ino; // ORDER DEPENDENCY: oset snapid_t snapid; - ino_t faked_ino; + ino_t faked_ino = 0; - uint32_t rdev; // if special file + uint32_t rdev = 0; // if special file // affected by any inode change... utime_t ctime; // inode change time utime_t btime; // birth time // perm (namespace permissions) - uint32_t mode; - uid_t uid; - gid_t gid; + uint32_t mode = 0; + uid_t uid = 0; + gid_t gid = 0; // nlink - int32_t nlink; + int32_t nlink = 0; // file (data access) - ceph_dir_layout dir_layout; + ceph_dir_layout dir_layout{}; file_layout_t layout; - uint64_t size; // on directory, # dentries - uint32_t truncate_seq; - uint64_t truncate_size; + uint64_t size = 0; // on directory, # dentries + uint32_t truncate_seq = 1; + uint64_t truncate_size = -1; utime_t mtime; // file data modify time. utime_t atime; // file data access time. - uint32_t time_warp_seq; // count of (potential) mtime/atime timewarps (i.e., utimes()) - uint64_t change_attr; + uint32_t time_warp_seq = 0; // count of (potential) mtime/atime timewarps (i.e., utimes()) + uint64_t change_attr = 0; - uint64_t max_size; // max size we can write to + uint64_t max_size = 0; // max size we can write to // dirfrag, recursive accountin frag_info_t dirstat; nest_info_t rstat; // special stuff - version_t version; // auth only - version_t xattr_version; + version_t version = 0; // auth only + version_t xattr_version = 0; utime_t snap_btime; // snapshot creation (birth) time std::map snap_metadata; // inline data - version_t inline_version; + version_t inline_version = 0; bufferlist inline_data; bool fscrypt = false; // fscrypt enabled ? - bool is_root() const { return ino == MDS_INO_ROOT; } + bool is_root() const { return ino == CEPH_INO_ROOT; } bool is_symlink() const { return (mode & S_IFMT) == S_IFLNK; } bool is_dir() const { return (mode & S_IFMT) == S_IFDIR; } bool is_file() const { return (mode & S_IFMT) == S_IFREG; } @@ -185,7 +182,7 @@ struct Inode { return ceph_str_hash(which, dn.data(), dn.length()); } - unsigned flags; + unsigned flags = 0; quota_info_t quota; @@ -195,24 +192,29 @@ struct Inode { } // about the dir (if this is one!) - Dir *dir; // if i'm a dir. + Dir *dir = 0; // if i'm a dir. fragtree_t dirfragtree; - uint64_t dir_release_count, dir_ordered_count; - bool dir_hashed, dir_replicated; + uint64_t dir_release_count = 1; + uint64_t dir_ordered_count = 1; + bool dir_hashed = false; + bool dir_replicated = false; // per-mds caps std::map caps; // mds -> Cap - Cap *auth_cap; - int64_t cap_dirtier_uid; - int64_t cap_dirtier_gid; - unsigned dirty_caps, flushing_caps; + Cap *auth_cap = 0; + int64_t cap_dirtier_uid = -1; + int64_t cap_dirtier_gid = -1; + unsigned dirty_caps = 0; + unsigned flushing_caps = 0; std::map flushing_cap_tids; - int shared_gen, cache_gen; - int snap_caps, snap_cap_refs; + int shared_gen = 0; + int cache_gen = 0; + int snap_caps = 0; + int snap_cap_refs = 0; utime_t hold_caps_until; xlist::item delay_cap_item, dirty_cap_item, flushing_cap_item; - SnapRealm *snaprealm; + SnapRealm *snaprealm = 0; xlist::item snaprealm_item; InodeRef snapdir_parent; // only if we are a snapdir inode map cap_snaps; // pending flush to mds @@ -223,10 +225,11 @@ struct Inode { ObjectCacher::ObjectSet oset; // ORDER DEPENDENCY: ino - uint64_t reported_size, wanted_max_size, requested_max_size; + uint64_t reported_size = 0; + uint64_t wanted_max_size = 0; + uint64_t requested_max_size = 0; - int _ref; // ref count. 1 for each dentry, fh that links to me. - uint64_t ll_ref; // separate ref count for ll client + uint64_t ll_ref = 0; // separate ref count for ll client xlist dentries; // if i'm linked to a dentry. string symlink; // symlink content, if it's a symlink map xattrs; @@ -246,12 +249,10 @@ struct Inode { void make_short_path(filepath& p); void make_nosnap_relative_path(filepath& p); - void get(); - int _put(int n=1); - - int get_num_ref() { - return _ref; - } + // The ref count. 1 for each dentry, fh, inode_map, + // cwd that links to me. + void iget() { get(); } + void iput(int n=1) { ceph_assert(n >= 0); while (n--) put(); } void ll_get() { ll_ref++; @@ -277,27 +278,13 @@ struct Inode { std::set fhs; - mds_rank_t dir_pin; + mds_rank_t dir_pin = MDS_RANK_NONE; + Inode() = delete; Inode(Client *c, vinodeno_t vino, file_layout_t *newlayout) - : client(c), ino(vino.ino), snapid(vino.snapid), faked_ino(0), - rdev(0), mode(0), uid(0), gid(0), nlink(0), - size(0), truncate_seq(1), truncate_size(-1), - time_warp_seq(0), change_attr(0), max_size(0), version(0), - xattr_version(0), inline_version(0), flags(0), - dir(0), dir_release_count(1), dir_ordered_count(1), - dir_hashed(false), dir_replicated(false), auth_cap(NULL), - cap_dirtier_uid(-1), cap_dirtier_gid(-1), - dirty_caps(0), flushing_caps(0), shared_gen(0), cache_gen(0), - snap_caps(0), snap_cap_refs(0), - delay_cap_item(this), dirty_cap_item(this), flushing_cap_item(this), - snaprealm(0), snaprealm_item(this), - oset((void *)this, newlayout->pool_id, this->ino), - reported_size(0), wanted_max_size(0), requested_max_size(0), - _ref(0), ll_ref(0), dir_pin(MDS_RANK_NONE) - { - memset(&dir_layout, 0, sizeof(dir_layout)); - } + : client(c), ino(vino.ino), snapid(vino.snapid), delay_cap_item(this), + dirty_cap_item(this), flushing_cap_item(this), snaprealm_item(this), + oset((void *)this, newlayout->pool_id, this->ino) {} ~Inode(); vinodeno_t vino() const { return vinodeno_t(ino, snapid); } diff --git a/ceph/src/cls/fifo/cls_fifo.cc b/ceph/src/cls/fifo/cls_fifo.cc index fc89a20e6..14313a735 100644 --- a/ceph/src/cls/fifo/cls_fifo.cc +++ b/ceph/src/cls/fifo/cls_fifo.cc @@ -217,7 +217,8 @@ int create_meta(cls_method_context_t hctx, auto iter = in->cbegin(); decode(op, iter); } catch (const ceph::buffer::error& err) { - CLS_ERR("ERROR: %s: failed to decode request", __PRETTY_FUNCTION__); + CLS_ERR("ERROR: %s: failed to decode request: %s", __PRETTY_FUNCTION__, + err.what()); return -EINVAL; } @@ -237,19 +238,24 @@ int create_meta(cls_method_context_t hctx, int r = cls_cxx_stat2(hctx, &size, nullptr); if (r < 0 && r != -ENOENT) { - CLS_ERR("ERROR: %s: cls_cxx_stat2() on obj returned %d", __PRETTY_FUNCTION__, r); + CLS_ERR("ERROR: %s: cls_cxx_stat2() on obj returned %d", + __PRETTY_FUNCTION__, r); return r; } if (op.exclusive && r == 0) { - CLS_ERR("%s: exclusive create but queue already exists", __PRETTY_FUNCTION__); + CLS_ERR("%s: exclusive create but queue already exists", + __PRETTY_FUNCTION__); return -EEXIST; } if (r == 0) { + CLS_LOG(5, "%s: FIFO already exists, reading from disk and comparing.", + __PRETTY_FUNCTION__); ceph::buffer::list bl; r = cls_cxx_read2(hctx, 0, size, &bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); if (r < 0) { - CLS_ERR("ERROR: %s: cls_cxx_read2() on obj returned %d", __PRETTY_FUNCTION__, r); + CLS_ERR("ERROR: %s: cls_cxx_read2() on obj returned %d", + __PRETTY_FUNCTION__, r); return r; } @@ -258,7 +264,8 @@ int create_meta(cls_method_context_t hctx, auto iter = bl.cbegin(); decode(header, iter); } catch (const ceph::buffer::error& err) { - CLS_ERR("ERROR: %s: failed decoding header", __PRETTY_FUNCTION__); + CLS_ERR("ERROR: %s: failed decoding header: %s", + __PRETTY_FUNCTION__, err.what()); return -EIO; } diff --git a/ceph/src/cls/rgw/cls_rgw.cc b/ceph/src/cls/rgw/cls_rgw.cc index 57ecdc692..bc455a04b 100644 --- a/ceph/src/cls/rgw/cls_rgw.cc +++ b/ceph/src/cls/rgw/cls_rgw.cc @@ -56,6 +56,9 @@ static std::string bucket_index_prefixes[] = { "", /* special handling for the o /* this must be the last index */ "9999_",}; +static const std::string BI_PREFIX_END = string(1, BI_PREFIX_CHAR) + + bucket_index_prefixes[BI_BUCKET_LAST_INDEX]; + static bool bi_is_objs_index(const string& s) { return ((unsigned char)s[0] != BI_PREFIX_CHAR); } @@ -2442,18 +2445,14 @@ static int rgw_bi_put_op(cls_method_context_t hctx, bufferlist *in, bufferlist * } static int list_plain_entries(cls_method_context_t hctx, - const string& name, - const string& marker, - uint32_t max, + const string& filter, + const string& start_after_key, + const string& end_key, + uint32_t max, list *entries, - bool *pmore) + bool *end_key_reached, + bool *pmore) { - string filter = name; - string start_after_key = marker; - - string end_key; // stop listing at bi_log_prefix - bi_log_prefix(end_key); - int count = 0; map keys; int ret = cls_cxx_map_get_vals(hctx, start_after_key, filter, max, @@ -2462,12 +2461,12 @@ static int list_plain_entries(cls_method_context_t hctx, return ret; } + *end_key_reached = false; + for (auto iter = keys.begin(); iter != keys.end(); ++iter) { - if (iter->first >= end_key) { - /* past the end of plain namespace */ - if (pmore) { - *pmore = false; - } + if (!end_key.empty() && iter->first >= end_key) { + *end_key_reached = true; + *pmore = true; return count; } @@ -2486,13 +2485,12 @@ static int list_plain_entries(cls_method_context_t hctx, return -EIO; } - CLS_LOG(20, "%s(): entry.idx=%s e.key.name=%s", __func__, escape_str(entry.idx).c_str(), escape_str(e.key.name).c_str()); + CLS_LOG(20, "%s(): entry.idx=%s e.key.name=%s", __func__, + escape_str(entry.idx).c_str(), escape_str(e.key.name).c_str()); - if (!name.empty() && e.key.name != name) { + if (!filter.empty() && e.key.name != filter) { /* we are skipping the rest of the entries */ - if (pmore) { - *pmore = false; - } + *pmore = false; return count; } @@ -2501,12 +2499,54 @@ static int list_plain_entries(cls_method_context_t hctx, if (count >= (int)max) { return count; } - start_after_key = entry.idx; } return count; } +static int list_plain_entries(cls_method_context_t hctx, + const string& name, + const string& marker, + uint32_t max, + list *entries, + bool *pmore) { + string start_after_key = marker; + string end_key; + bi_log_prefix(end_key); + int r; + bool end_key_reached; + bool more; + + if (start_after_key < end_key) { + // listing ascii plain namespace + int r = list_plain_entries(hctx, name, start_after_key, end_key, max, + entries, &end_key_reached, &more); + if (r < 0) { + return r; + } + if (r >= (int)max || !end_key_reached || !more) { + if (pmore) { + *pmore = more; + } + return r; + } + start_after_key = BI_PREFIX_END; + max = max - r; + } + + // listing non-ascii plain namespace + r = list_plain_entries(hctx, name, start_after_key, {}, max, entries, + &end_key_reached, &more); + if (r < 0) { + return r; + } + if (pmore) { + *pmore = more; + } + + return r; +} + static int list_instance_entries(cls_method_context_t hctx, const string& name, const string& marker, diff --git a/ceph/src/common/buffer.cc b/ceph/src/common/buffer.cc index 8d7583e7a..406ca24a4 100644 --- a/ceph/src/common/buffer.cc +++ b/ceph/src/common/buffer.cc @@ -99,8 +99,8 @@ static ceph::spinlock debug_lock; unsigned align, int mempool = mempool::mempool_buffer_anon) { - if (!align) - align = sizeof(size_t); + // posix_memalign() requires a multiple of sizeof(void *) + align = std::max(align, sizeof(void *)); size_t rawlen = round_up_to(sizeof(buffer::raw_combined), alignof(buffer::raw_combined)); size_t datalen = round_up_to(len, alignof(buffer::raw_combined)); @@ -161,8 +161,8 @@ static ceph::spinlock debug_lock; MEMPOOL_CLASS_HELPERS(); raw_posix_aligned(unsigned l, unsigned _align) : raw(l) { - align = _align; - ceph_assert((align >= sizeof(void *)) && (align & (align - 1)) == 0); + // posix_memalign() requires a multiple of sizeof(void *) + align = std::max(_align, sizeof(void *)); #ifdef DARWIN data = (char *) valloc(len); #else diff --git a/ceph/src/common/config.cc b/ceph/src/common/config.cc index b110e9977..491685867 100644 --- a/ceph/src/common/config.cc +++ b/ceph/src/common/config.cc @@ -361,7 +361,6 @@ int md_config_t::parse_config_files(ConfigValues& values, values.cluster = get_cluster_name(nullptr); } // open new conf - string conffile; for (auto& fn : get_conffile_paths(values, conf_files_str, warnings, flags)) { bufferlist bl; std::string error; @@ -373,7 +372,7 @@ int md_config_t::parse_config_files(ConfigValues& values, int ret = parse_buffer(values, tracker, bl.c_str(), bl.length(), &oss); if (ret == 0) { parse_error.clear(); - conffile = fn; + conf_path = fn; break; } parse_error = oss.str(); @@ -382,11 +381,11 @@ int md_config_t::parse_config_files(ConfigValues& values, } } // it must have been all ENOENTs, that's the only way we got here - if (conffile.empty()) { + if (conf_path.empty()) { return -ENOENT; } if (values.cluster.empty()) { - values.cluster = get_cluster_name(conffile.c_str()); + values.cluster = get_cluster_name(conf_path.c_str()); } update_legacy_vals(values); return 0; diff --git a/ceph/src/common/config.h b/ceph/src/common/config.h index bb3410e61..ef7d5b34f 100644 --- a/ceph/src/common/config.h +++ b/ceph/src/common/config.h @@ -319,10 +319,15 @@ public: // for global_init const char *conf_files, std::ostream *warnings, int flags) const; + + const std::string& get_conf_path() const { + return conf_path; + } private: static std::string get_cluster_name(const char* conffile_path); // The configuration file we read, or NULL if we haven't read one. ConfFile cf; + std::string conf_path; public: std::string parse_error; private: diff --git a/ceph/src/common/config_proxy.h b/ceph/src/common/config_proxy.h index 0cf539352..cb30a2d7f 100644 --- a/ceph/src/common/config_proxy.h +++ b/ceph/src/common/config_proxy.h @@ -341,6 +341,9 @@ public: std::lock_guard l{lock}; config.get_defaults_bl(values, bl); } + const std::string& get_conf_path() const { + return config.get_conf_path(); + } }; } diff --git a/ceph/src/common/dout.h b/ceph/src/common/dout.h index b8f762991..421222d53 100644 --- a/ceph/src/common/dout.h +++ b/ceph/src/common/dout.h @@ -175,6 +175,11 @@ struct is_dynamic> : public std::true_type {}; #define ldout(cct, v) dout_impl(cct, dout_subsys, v) dout_prefix #define lderr(cct) dout_impl(cct, ceph_subsys_, -1) dout_prefix +#define ldpp_subdout(dpp, sub, v) \ + if (decltype(auto) pdpp = (dpp); pdpp) /* workaround -Wnonnull-compare for 'this' */ \ + dout_impl(pdpp->get_cct(), ceph_subsys_##sub, v) \ + pdpp->gen_prefix(*_dout) + #define ldpp_dout(dpp, v) \ if (decltype(auto) pdpp = (dpp); pdpp) /* workaround -Wnonnull-compare for 'this' */ \ dout_impl(pdpp->get_cct(), ceph::dout::need_dynamic(pdpp->get_subsys()), v) \ diff --git a/ceph/src/common/options.cc b/ceph/src/common/options.cc index 5e640acd6..96eabd2d9 100644 --- a/ceph/src/common/options.cc +++ b/ceph/src/common/options.cc @@ -3204,18 +3204,22 @@ std::vector