From 1911f103e16ae0d04db10fb41db8217ef4c320d3 Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Fri, 22 May 2020 15:33:08 +0200 Subject: [PATCH] import 15.2.2 octopus source Signed-off-by: Thomas Lamprecht --- ceph/.github/CODEOWNERS | 7 +- ceph/CMakeLists.txt | 4 +- ceph/PendingReleaseNotes | 2 + ceph/alpine/APKBUILD | 6 +- ceph/ceph.spec | 35 +- ceph/ceph.spec.in | 29 +- ceph/changelog.upstream | 10 +- ceph/cmake/modules/FindFUSE.cmake | 58 + ceph/cmake/modules/Findfuse.cmake | 28 - ceph/debian/cephadm.install | 1 + ceph/doc/cephadm/client-setup.rst | 36 + ceph/doc/cephadm/drivegroups.rst | 2 +- ceph/doc/cephadm/index.rst | 1 + ceph/doc/cephadm/install.rst | 15 +- ceph/doc/cephadm/troubleshooting.rst | 46 + ceph/doc/cephadm/upgrade.rst | 6 +- ceph/doc/cephfs/fs-volumes.rst | 21 + ceph/doc/dev/cephadm.rst | 87 + ceph/doc/install/index.rst | 2 +- ceph/doc/man/8/CMakeLists.txt | 3 +- ceph/doc/man/8/cephadm.rst | 441 +++++ ceph/doc/mgr/orchestrator.rst | 19 +- ceph/doc/releases/nautilus.rst | 3 + .../grafana/dashboards/ceph-cluster.json | 1 + .../grafana/dashboards/host-details.json | 14 +- .../grafana/dashboards/hosts-overview.json | 8 +- .../dashboards/osd-device-details.json | 14 +- .../grafana/dashboards/pool-detail.json | 5 +- .../prometheus/alerts/ceph_default_alerts.yml | 19 +- ceph/qa/cephfs/begin.yaml | 10 + .../supported-random-distro$/centos_8.yaml | 1 - .../centos_latest.yaml | 1 + .../{rhel_8.yaml => rhel_latest.yaml} | 0 ceph/qa/distros/supported/centos_latest.yaml | 2 +- ceph/qa/overrides/short_pg_log.yaml | 1 + ceph/qa/rgw_bucket_sharding/many.yaml | 5 - .../qa/standalone/mon/mon-last-epoch-clean.sh | 307 ++++ ceph/qa/standalone/osd/osd-backfill-stats.sh | 7 + ceph/qa/standalone/scrub/osd-scrub-repair.sh | 3 +- .../tasks/cephfs_scrub_tests.yaml | 1 + .../tasks/cfuse_workunit_suites_fsx.yaml | 12 - .../tasks/kclient_workunit_suites_fsx.yaml | 12 - .../tasks/cfuse_workunit_suites_fsx.yaml | 12 - ceph/qa/suites/rados/cephadm/smoke-roleless/% | 0 .../suites/rados/cephadm/smoke-roleless/.qa | 1 + .../rados/cephadm/smoke-roleless/distro | 1 + .../rados/cephadm/smoke-roleless/start.yaml | 32 + .../suites/rados/cephadm/upgrade/1-start.yaml | 4 +- .../rados/dashboard/tasks/dashboard.yaml | 1 + .../rados/mgr/tasks/module_selftest.yaml | 1 + ceph/qa/suites/rados/rest/centos_latest.yaml | 1 - .../rados/rest/supported-random-distro$ | 1 + .../rados/singleton/all/osd-recovery.yaml | 1 + .../radosbench-high-concurrency.yaml | 49 + ceph/qa/suites/rgw/crypt/3-rgw/rgw.yaml | 2 + ceph/qa/suites/rgw/hadoop-s3a/overrides.yaml | 6 + ceph/qa/suites/rgw/multifs/overrides.yaml | 2 + ceph/qa/suites/rgw/multisite/overrides.yaml | 2 + ceph/qa/suites/rgw/singleton/overrides.yaml | 2 + ceph/qa/suites/rgw/tempest/overrides.yaml | 6 + ceph/qa/suites/rgw/thrash/civetweb.yaml | 5 + ceph/qa/suites/rgw/verify/overrides.yaml | 2 + ceph/qa/suites/rgw/website/overrides.yaml | 2 + .../octopus-client-x/.qa | 1 + .../octopus-client-x/rbd/% | 0 .../octopus-client-x/rbd/.qa | 1 + .../octopus-client-x/rbd/0-cluster/+ | 0 .../octopus-client-x/rbd/0-cluster/.qa | 1 + .../rbd/0-cluster/openstack.yaml | 4 + .../octopus-client-x/rbd/0-cluster/start.yaml | 21 + .../octopus-client-x/rbd/1-install/.qa | 1 + .../rbd/1-install/nautilus-client-x.yaml | 11 + .../octopus-client-x/rbd/2-features/.qa | 1 + .../rbd/2-features/defaults.yaml | 6 + .../rbd/2-features/layering.yaml | 6 + .../octopus-client-x/rbd/3-workload/.qa | 1 + .../3-workload/rbd_notification_tests.yaml | 36 + .../octopus-client-x/rbd/supported/.qa | 1 + .../rbd/supported/ubuntu_18.04.yaml | 1 + .../octopus-p2p/octopus-p2p-parallel/% | 0 .../point-to-point-upgrade.yaml | 177 ++ .../supported-all-distro/ubuntu_latest.yaml | 2 + .../octopus-p2p/octopus-p2p-stress-split/% | 0 .../octopus-p2p-stress-split/0-cluster/+ | 0 .../0-cluster/openstack.yaml | 6 + .../0-cluster/start.yaml | 33 + .../1-ceph-install/octopus.yaml | 19 + .../1.1.short_pg_log.yaml | 6 + .../2-partial-upgrade/firsthalf.yaml | 13 + .../3-thrash/default.yaml | 27 + .../octopus-p2p-stress-split/4-workload/+ | 0 .../4-workload/fsx.yaml | 8 + .../4-workload/radosbench.yaml | 52 + .../4-workload/rbd-cls.yaml | 10 + .../4-workload/rbd-import-export.yaml | 12 + .../4-workload/rbd_api.yaml | 10 + .../4-workload/readwrite.yaml | 16 + .../4-workload/snaps-few-objects.yaml | 18 + .../5-finish-upgrade.yaml | 8 + .../6-final-workload/+ | 0 .../6-final-workload/rbd-python.yaml | 10 + .../6-final-workload/snaps-many-objects.yaml | 16 + .../objectstore/bluestore-bitmap.yaml | 43 + .../objectstore/bluestore-comp.yaml | 23 + .../objectstore/bluestore-stupid.yaml | 43 + .../objectstore/filestore-xfs.yaml | 15 + .../supported-all-distro/ubuntu_latest.yaml | 2 + .../thrashosds-health.yaml | 15 + ceph/qa/tasks/cephadm.py | 149 +- ceph/qa/tasks/cephfs/fuse_mount.py | 3 + ceph/qa/tasks/cephfs/kernel_mount.py | 3 + ceph/qa/tasks/cephfs/test_admin.py | 38 + ceph/qa/tasks/cephfs/test_data_scan.py | 13 + ceph/qa/tasks/cephfs/test_full.py | 57 +- ceph/qa/tasks/cephfs/test_scrub_checks.py | 30 +- ceph/qa/tasks/cephfs/test_volumes.py | 108 ++ ceph/qa/tasks/cephfs/xfstests_dev.py | 4 +- .../tasks/mgr/dashboard/test_perf_counters.py | 2 +- ceph/qa/tasks/mgr/dashboard/test_rgw.py | 6 +- ceph/qa/tasks/radosbench.py | 3 + ceph/qa/tasks/rgw.py | 5 +- ceph/qa/tasks/vstart_runner.py | 42 +- ceph/qa/workunits/cephadm/test_adoption.sh | 5 +- ceph/qa/workunits/cephadm/test_cephadm.sh | 71 +- ceph/qa/workunits/mon/pool_ops.sh | 38 + ceph/qa/workunits/rbd/rbd_mirror_ha.sh | 2 +- ceph/qa/workunits/rbd/rbd_mirror_helpers.sh | 48 +- ceph/qa/workunits/rbd/rbd_mirror_journal.sh | 46 +- ceph/qa/workunits/rest/test_mgr_rest_api.py | 2 +- ceph/src/.git_version | 4 +- ceph/src/CMakeLists.txt | 5 +- .../ceph_volume/devices/lvm/batch.py | 16 +- .../ceph_volume/devices/simple/scan.py | 5 +- .../batch/playbooks/test_explicit.yml | 6 +- .../tests/functional/playbooks/deploy.yml | 2 + ceph/src/ceph.in | 2 +- ceph/src/ceph_fuse.cc | 11 + ceph/src/cephadm/cephadm | 409 ++++- ceph/src/cephadm/samples/rgw_ssl.json | 103 ++ ceph/src/cephadm/tox.ini | 2 +- ceph/src/client/Client.cc | 48 +- ceph/src/client/fuse_ll.cc | 80 +- ceph/src/cls/journal/cls_journal_types.h | 3 + ceph/src/cls/rbd/cls_rbd_types.cc | 12 +- ceph/src/cls/rbd/cls_rbd_types.h | 5 +- ceph/src/common/AsyncOpTracker.h | 22 +- ceph/src/common/options.cc | 7 +- ceph/src/include/ceph_fuse.h | 32 + ceph/src/include/denc.h | 2 +- ceph/src/include/rbd/librbd.h | 1 + ceph/src/kv/RocksDBStore.h | 3 - ceph/src/librbd/DeepCopyRequest.cc | 6 +- ceph/src/librbd/DeepCopyRequest.h | 9 +- ceph/src/librbd/api/Image.cc | 8 +- ceph/src/librbd/api/Migration.cc | 14 +- ceph/src/librbd/api/Mirror.cc | 34 +- ceph/src/librbd/api/Trash.cc | 4 +- ceph/src/librbd/deep_copy/Handler.h | 50 + ceph/src/librbd/deep_copy/ImageCopyRequest.cc | 11 +- ceph/src/librbd/deep_copy/ImageCopyRequest.h | 11 +- .../src/librbd/deep_copy/ObjectCopyRequest.cc | 16 +- ceph/src/librbd/deep_copy/ObjectCopyRequest.h | 10 +- ceph/src/librbd/image/CloneRequest.cc | 17 +- ceph/src/librbd/image/CreateRequest.cc | 21 +- ceph/src/librbd/image/CreateRequest.h | 9 +- ceph/src/librbd/image/Types.h | 20 + ceph/src/librbd/internal.cc | 19 +- ceph/src/librbd/io/CopyupRequest.cc | 2 +- ceph/src/librbd/mirror/EnableRequest.cc | 84 +- ceph/src/librbd/mirror/EnableRequest.h | 31 +- .../snapshot/CreateNonPrimaryRequest.cc | 66 + .../mirror/snapshot/CreateNonPrimaryRequest.h | 13 +- .../mirror/snapshot/CreatePrimaryRequest.cc | 10 +- .../mirror/snapshot/CreatePrimaryRequest.h | 8 +- .../librbd/mirror/snapshot/DemoteRequest.cc | 2 +- .../librbd/mirror/snapshot/PromoteRequest.cc | 2 +- .../mirror/snapshot/UnlinkPeerRequest.cc | 10 +- .../librbd/operation/EnableFeaturesRequest.cc | 4 +- ceph/src/librbd/operation/MigrateRequest.cc | 2 +- ceph/src/mds/Locker.cc | 10 +- ceph/src/mds/MDCache.cc | 2 +- ceph/src/mds/Mutation.cc | 7 +- ceph/src/mds/Mutation.h | 3 +- ceph/src/mds/SimpleLock.cc | 12 +- ceph/src/mds/SimpleLock.h | 2 +- ceph/src/mgr/DaemonServer.cc | 102 +- ceph/src/mgr/DaemonServer.h | 1 + ceph/src/mgr/PyModule.cc | 3 + ceph/src/mon/FSCommands.cc | 6 +- ceph/src/mon/Monitor.cc | 56 +- ceph/src/mon/OSDMonitor.cc | 89 +- ceph/src/mon/OSDMonitor.h | 5 +- ceph/src/os/CMakeLists.txt | 3 +- ceph/src/os/FuseStore.cc | 91 +- .../src/os/bluestore/BitmapFreelistManager.cc | 197 +- ceph/src/os/bluestore/BitmapFreelistManager.h | 21 +- ceph/src/os/bluestore/BlueFS.cc | 33 +- ceph/src/os/bluestore/BlueFS.h | 16 +- ceph/src/os/bluestore/BlueRocksEnv.cc | 2 +- ceph/src/os/bluestore/BlueStore.cc | 194 +- ceph/src/os/bluestore/BlueStore.h | 15 +- ceph/src/os/bluestore/FreelistManager.h | 13 +- ceph/src/os/bluestore/bluestore_tool.cc | 5 +- ceph/src/os/bluestore/bluestore_types.h | 3 + ceph/src/osd/PeeringState.cc | 5 +- ceph/src/osd/PeeringState.h | 4 + ceph/src/osd/PrimaryLogPG.cc | 3 +- ceph/src/osd/PrimaryLogPG.h | 3 + ceph/src/pybind/mgr/cephadm/module.py | 391 ++-- ceph/src/pybind/mgr/cephadm/nfs.py | 53 +- ceph/src/pybind/mgr/cephadm/tests/fixtures.py | 2 +- .../pybind/mgr/cephadm/tests/test_cephadm.py | 396 ++-- .../src/pybind/mgr/cephadm/tests/test_spec.py | 290 +++ ceph/src/pybind/mgr/cephadm/utils.py | 2 +- .../pybind/mgr/dashboard/controllers/iscsi.py | 70 +- .../pybind/mgr/dashboard/controllers/rbd.py | 7 +- .../dist/en-US/2.adb5a86b21cf4e105cd2.js | 1 - .../dist/en-US/2.d36650ee0a92dfd05faa.js | 1 + .../dist/en-US/6.4299ba94dd69b0b04046.js | 1 + .../dist/en-US/6.fd03afd7d5e918275ed6.js | 1 - .../dist/en-US/7.b9f7cf0935ea27ecc60a.js | 1 - .../dist/en-US/7.cf5f8c70f123c771366b.js | 1 + ...10d3e6387.js => 8.f45b72794d78f44d11b9.js} | 2 +- .../dist/en-US/9.827ca6d4c4df11fe3a62.js | 1 - .../dist/en-US/9.a6e2f7400e0a62470111.js | 1 + .../dashboard/frontend/dist/en-US/index.html | 2 +- .../dist/en-US/main.3101f12ca40b61271b1f.js | 1 - .../dist/en-US/main.bbd2d62de0d7a670cae3.js | 1 + .../en-US/runtime.2fd4e31a298ae797b302.js | 1 + .../en-US/runtime.a1d3bd9700d820216e9a.js | 1 - .../mgr/dashboard/frontend/i18n.config.json | 2 +- ...scsi-target-discovery-modal.component.html | 16 +- ...i-target-discovery-modal.component.spec.ts | 20 +- .../iscsi-target-discovery-modal.component.ts | 4 +- .../iscsi-target-form.component.html | 32 +- .../iscsi-target-form.component.spec.ts | 30 +- .../iscsi-target-form.component.ts | 4 +- .../mirroring/overview/overview.component.ts | 24 +- .../block/rbd-form/rbd-form.component.spec.ts | 52 +- .../ceph/block/rbd-form/rbd-form.component.ts | 8 +- .../rbd-trash-purge-modal.component.spec.ts | 1 - .../rules-list/rules-list.component.html | 16 +- .../rules-list/rules-list.component.spec.ts | 4 +- .../service-daemon-list.component.ts | 13 +- .../service-details.component.spec.ts | 15 +- .../services/services.component.spec.ts | 26 +- .../cluster/services/services.component.ts | 23 +- .../nfs/nfs-form/nfs-form.component.spec.ts | 1 - .../nfs/nfs-list/nfs-list.component.spec.ts | 7 +- .../pool-details.component.spec.ts | 13 +- .../rgw-bucket-list.component.html | 4 + .../rgw-bucket-list.component.ts | 21 +- .../rgw-user-list.component.html | 4 + .../rgw-user-list/rgw-user-list.component.ts | 22 +- .../workbench-layout.component.ts | 25 +- .../navigation/navigation.component.ts | 33 +- .../notifications/notifications.component.ts | 26 +- .../app/shared/api/ceph-service.service.ts | 6 +- .../shared/api/rbd-mirroring.service.spec.ts | 54 +- .../app/shared/api/rbd-mirroring.service.ts | 39 +- .../notifications-sidebar.component.ts | 104 +- .../datatable/table/table.component.html | 7 + .../shared/datatable/table/table.component.ts | 3 + .../src/app/shared/enum/cell-template.enum.ts | 13 +- .../app/shared/models/service.interface.ts | 13 +- .../src/app/shared/pipes/pipes.module.ts | 10 +- .../app/shared/pipes/truncate.pipe.spec.ts | 21 + .../src/app/shared/pipes/truncate.pipe.ts | 16 + .../services/feature-toggles.service.ts | 14 +- .../shared/services/summary.service.spec.ts | 23 +- .../app/shared/services/summary.service.ts | 42 +- .../services/task-manager.service.spec.ts | 1 + .../shared/services/task-manager.service.ts | 6 +- .../app/shared/services/timer.service.spec.ts | 68 + .../src/app/shared/services/timer.service.ts | 27 + .../frontend/src/locale/messages.cs.xlf | 1588 ++++++++++++---- .../frontend/src/locale/messages.de-DE.xlf | 1635 +++++++++++++---- .../frontend/src/locale/messages.es-ES.xlf | 1554 ++++++++++++---- .../frontend/src/locale/messages.fr-FR.xlf | 1554 ++++++++++++---- .../frontend/src/locale/messages.id-ID.xlf | 1554 ++++++++++++---- .../frontend/src/locale/messages.it-IT.xlf | 1554 ++++++++++++---- .../frontend/src/locale/messages.ja-JP.xlf | 1554 ++++++++++++---- .../frontend/src/locale/messages.ko-KR.xlf | 1553 ++++++++++++---- .../frontend/src/locale/messages.pl-PL.xlf | 1552 ++++++++++++---- .../frontend/src/locale/messages.pt-BR.xlf | 1554 ++++++++++++---- .../frontend/src/locale/messages.zh-CN.xlf | 1554 ++++++++++++---- .../frontend/src/locale/messages.zh-TW.xlf | 1562 ++++++++++++---- .../frontend/src/testing/unit-test-helper.ts | 20 + ceph/src/pybind/mgr/dashboard/module.py | 2 +- .../src/pybind/mgr/dashboard/plugins/debug.py | 2 +- .../pybind/mgr/dashboard/tests/test_iscsi.py | 58 +- .../src/pybind/mgr/orchestrator/_interface.py | 119 +- ceph/src/pybind/mgr/orchestrator/module.py | 272 ++- ceph/src/pybind/mgr/requirements.txt | 2 + ceph/src/pybind/mgr/rook/module.py | 32 +- ceph/src/pybind/mgr/telegraf/module.py | 10 +- .../mgr/test_orchestrator/dummy_data.json | 494 +++-- .../pybind/mgr/test_orchestrator/module.py | 16 +- ceph/src/pybind/mgr/tests/__init__.py | 1 - ceph/src/pybind/mgr/tox.ini | 22 +- .../fs/operations/versions/subvolume_base.py | 25 + ceph/src/pybind/mgr/volumes/fs/volume.py | 23 + ceph/src/pybind/mgr/volumes/module.py | 14 + ceph/src/pybind/rbd/rbd.pyx | 151 +- .../ceph/deployment/drive_group.py | 16 +- .../ceph/deployment/service_spec.py | 169 +- .../ceph/deployment/translate.py | 24 +- .../ceph/tests/test_drive_group.py | 48 +- .../ceph/tests/test_service_spec.py | 50 +- ceph/src/python-common/requirements.txt | 1 + ceph/src/rbd_fuse/CMakeLists.txt | 2 +- ceph/src/rbd_fuse/rbd-fuse.cc | 57 +- ceph/src/rgw/rgw_admin.cc | 11 +- ceph/src/rgw/rgw_bucket.cc | 39 + ceph/src/rgw/rgw_bucket.h | 2 + ceph/src/rgw/rgw_main.cc | 17 +- ceph/src/rgw/rgw_op.cc | 16 +- ceph/src/rgw/rgw_reshard.cc | 23 +- ceph/src/rgw/rgw_sync_module_pubsub.cc | 2 +- ceph/src/test/cli-integration/rbd/unmap.t | 184 +- ceph/src/test/cli/rbd/help.t | 12 + ceph/src/test/libcephfs/deleg.cc | 30 +- .../deep_copy/test_mock_ImageCopyRequest.cc | 38 +- .../deep_copy/test_mock_ObjectCopyRequest.cc | 2 +- .../librbd/image/test_mock_CloneRequest.cc | 7 +- .../test/librbd/io/test_mock_CopyupRequest.cc | 2 +- .../test_mock_CreateNonPrimaryRequest.cc | 81 + .../test_mock_CreatePrimaryRequest.cc | 20 +- .../snapshot/test_mock_PromoteRequest.cc | 1 + .../librbd/mirror/snapshot/test_mock_Utils.cc | 1 - .../test_mock_EnableFeaturesRequest.cc | 3 +- ceph/src/test/librbd/test_librbd.cc | 42 + .../test/librbd/test_mock_DeepCopyRequest.cc | 17 +- ceph/src/test/objectstore/store_test.cc | 93 + ceph/src/test/objectstore/test_bluefs.cc | 10 +- .../test/objectstore/test_bluestore_types.cc | 92 + ceph/src/test/pybind/test_rbd.py | 9 +- .../journal/test_mock_Replayer.cc | 1 + .../snapshot/test_mock_Replayer.cc | 392 +++- .../test/rbd_mirror/test_mock_ImageSync.cc | 3 +- .../rbd_mirror/test_mock_InstanceReplayer.cc | 14 +- ceph/src/tools/CMakeLists.txt | 2 +- ceph/src/tools/cephfs/JournalTool.cc | 19 +- ceph/src/tools/rbd/ArgumentTypes.cc | 17 +- ceph/src/tools/rbd/ArgumentTypes.h | 3 + ceph/src/tools/rbd/Utils.cc | 5 + ceph/src/tools/rbd/action/MirrorPool.cc | 51 +- ceph/src/tools/rbd/action/Snap.cc | 4 +- ceph/src/tools/rbd_mirror/CMakeLists.txt | 1 + ceph/src/tools/rbd_mirror/ImageReplayer.cc | 59 + ceph/src/tools/rbd_mirror/ImageReplayer.h | 7 + ceph/src/tools/rbd_mirror/ImageSync.cc | 16 +- ceph/src/tools/rbd_mirror/ImageSync.h | 5 +- ceph/src/tools/rbd_mirror/InstanceReplayer.cc | 29 +- .../image_replayer/CreateImageRequest.cc | 2 +- .../image_replayer/OpenImageRequest.cc | 5 + .../image_replayer/TimeRollingMean.cc | 34 + .../image_replayer/TimeRollingMean.h | 40 + .../journal/ReplayStatusFormatter.cc | 69 +- .../journal/ReplayStatusFormatter.h | 8 +- .../image_replayer/journal/Replayer.cc | 32 +- .../image_replayer/journal/Replayer.h | 1 - .../image_replayer/snapshot/Replayer.cc | 329 +++- .../image_replayer/snapshot/Replayer.h | 74 +- ceph/test_cephadm.sh | 100 - 365 files changed, 23190 insertions(+), 6372 deletions(-) create mode 100644 ceph/cmake/modules/FindFUSE.cmake delete mode 100644 ceph/cmake/modules/Findfuse.cmake create mode 100644 ceph/doc/cephadm/client-setup.rst create mode 100644 ceph/doc/dev/cephadm.rst create mode 100644 ceph/doc/man/8/cephadm.rst delete mode 120000 ceph/qa/distros/supported-random-distro$/centos_8.yaml create mode 120000 ceph/qa/distros/supported-random-distro$/centos_latest.yaml rename ceph/qa/distros/supported-random-distro$/{rhel_8.yaml => rhel_latest.yaml} (100%) delete mode 100644 ceph/qa/rgw_bucket_sharding/many.yaml create mode 100755 ceph/qa/standalone/mon/mon-last-epoch-clean.sh create mode 100644 ceph/qa/suites/rados/cephadm/smoke-roleless/% create mode 120000 ceph/qa/suites/rados/cephadm/smoke-roleless/.qa create mode 120000 ceph/qa/suites/rados/cephadm/smoke-roleless/distro create mode 100644 ceph/qa/suites/rados/cephadm/smoke-roleless/start.yaml delete mode 120000 ceph/qa/suites/rados/rest/centos_latest.yaml create mode 120000 ceph/qa/suites/rados/rest/supported-random-distro$ create mode 100644 ceph/qa/suites/rados/thrash/workloads/radosbench-high-concurrency.yaml create mode 100644 ceph/qa/suites/rgw/hadoop-s3a/overrides.yaml create mode 100644 ceph/qa/suites/rgw/tempest/overrides.yaml create mode 120000 ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/.qa create mode 100644 ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/% create mode 120000 ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/.qa create mode 100644 ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/+ create mode 120000 ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/.qa create mode 100644 ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/openstack.yaml create mode 100644 ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/start.yaml create mode 120000 ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/.qa create mode 100644 ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/nautilus-client-x.yaml create mode 120000 ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/.qa create mode 100644 ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/defaults.yaml create mode 100644 ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/layering.yaml create mode 120000 ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/.qa create mode 100644 ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/rbd_notification_tests.yaml create mode 120000 ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/.qa create mode 120000 ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/ubuntu_18.04.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-parallel/% create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-parallel/point-to-point-upgrade.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-parallel/supported-all-distro/ubuntu_latest.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/% create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/0-cluster/+ create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/0-cluster/openstack.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/0-cluster/start.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/1-ceph-install/octopus.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/1.1.short_pg_log.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/2-partial-upgrade/firsthalf.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/3-thrash/default.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/+ create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/fsx.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/radosbench.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/rbd-cls.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/rbd-import-export.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/rbd_api.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/readwrite.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/snaps-few-objects.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/5-finish-upgrade.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/6-final-workload/+ create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/6-final-workload/rbd-python.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/6-final-workload/snaps-many-objects.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/objectstore/bluestore-bitmap.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/objectstore/bluestore-comp.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/objectstore/bluestore-stupid.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/objectstore/filestore-xfs.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/supported-all-distro/ubuntu_latest.yaml create mode 100644 ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/thrashosds-health.yaml create mode 100644 ceph/src/cephadm/samples/rgw_ssl.json create mode 100644 ceph/src/include/ceph_fuse.h create mode 100644 ceph/src/librbd/deep_copy/Handler.h create mode 100644 ceph/src/librbd/image/Types.h create mode 100644 ceph/src/pybind/mgr/cephadm/tests/test_spec.py delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/2.adb5a86b21cf4e105cd2.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/2.d36650ee0a92dfd05faa.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/6.4299ba94dd69b0b04046.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/6.fd03afd7d5e918275ed6.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/7.b9f7cf0935ea27ecc60a.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/7.cf5f8c70f123c771366b.js rename ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/{8.50be3ebee1410d3e6387.js => 8.f45b72794d78f44d11b9.js} (54%) delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/9.827ca6d4c4df11fe3a62.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/9.a6e2f7400e0a62470111.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/main.3101f12ca40b61271b1f.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/main.bbd2d62de0d7a670cae3.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/runtime.2fd4e31a298ae797b302.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/runtime.a1d3bd9700d820216e9a.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/truncate.pipe.spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/truncate.pipe.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/services/timer.service.spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/services/timer.service.ts create mode 100644 ceph/src/tools/rbd_mirror/image_replayer/TimeRollingMean.cc create mode 100644 ceph/src/tools/rbd_mirror/image_replayer/TimeRollingMean.h delete mode 100755 ceph/test_cephadm.sh diff --git a/ceph/.github/CODEOWNERS b/ceph/.github/CODEOWNERS index 97bc99147..86e37fa11 100644 --- a/ceph/.github/CODEOWNERS +++ b/ceph/.github/CODEOWNERS @@ -14,12 +14,15 @@ /src/pybind/mgr/cephadm @ceph/orchestrators /src/pybind/mgr/test_orchestrator @ceph/orchestrators /src/python-common/ceph/deployment @ceph/orchestrators -/qa/workunits/cephadm/test_cephadm.sh @ceph/orchestrators +/qa/workunits/cephadm @ceph/orchestrators /qa/tasks/cephadm.py @ceph/orchestrators /qa/tasks/mgr/test_orchestrator_cli.py @ceph/orchestrators /qa/tasks/mgr/test_cephadm_orchestrator.py @ceph/orchestrators -/doc/mgr/orchestrator_cli.rst @ceph/orchestrators +/qa/suites/rados/cephadm @ceph/orchestrators +/doc/mgr/orchestrator.rst @ceph/orchestrators /doc/mgr/orchestrator_modules.rst @ceph/orchestrators +/doc/cephadm @ceph/orchestrators +/doc/dev/cephadm.rst @ceph/orchestrators #ceph-volume /src/ceph-volume @ceph/ceph-volume diff --git a/ceph/CMakeLists.txt b/ceph/CMakeLists.txt index b2dc184e6..caf4ac4dd 100644 --- a/ceph/CMakeLists.txt +++ b/ceph/CMakeLists.txt @@ -159,7 +159,7 @@ endif() option(WITH_FUSE "Fuse is here" ON) if(WITH_FUSE) - find_package(fuse) + find_package(FUSE) set(HAVE_LIBFUSE ${FUSE_FOUND}) endif() @@ -667,4 +667,4 @@ add_custom_target(tags DEPENDS ctags) find_package(CppCheck) find_package(IWYU) -set(VERSION 15.2.1) +set(VERSION 15.2.2) diff --git a/ceph/PendingReleaseNotes b/ceph/PendingReleaseNotes index c9fd4c794..6e07ce6d4 100644 --- a/ceph/PendingReleaseNotes +++ b/ceph/PendingReleaseNotes @@ -1,6 +1,8 @@ >=15.0.0 -------- +* CVE-2020-10736: Fixes an authorization bypass in monitor and manager daemons + * The RGW "num_rados_handles" has been removed. * If you were using a value of "num_rados_handles" greater than 1 multiply your current "objecter_inflight_ops" and diff --git a/ceph/alpine/APKBUILD b/ceph/alpine/APKBUILD index d3c76725b..78b1ffd01 100644 --- a/ceph/alpine/APKBUILD +++ b/ceph/alpine/APKBUILD @@ -1,7 +1,7 @@ # Contributor: John Coyle # Maintainer: John Coyle pkgname=ceph -pkgver=15.2.1 +pkgver=15.2.2 pkgrel=0 pkgdesc="Ceph is a distributed object store and file system" pkgusers="ceph" @@ -63,7 +63,7 @@ makedepends=" xmlstarlet yasm " -source="ceph-15.2.1.tar.bz2" +source="ceph-15.2.2.tar.bz2" subpackages=" $pkgname-base $pkgname-common @@ -116,7 +116,7 @@ _sysconfdir=/etc _udevrulesdir=/etc/udev/rules.d _python_sitelib=/usr/lib/python2.7/site-packages -builddir=$srcdir/ceph-15.2.1 +builddir=$srcdir/ceph-15.2.2 build() { export CEPH_BUILD_VIRTUALENV=$builddir diff --git a/ceph/ceph.spec b/ceph/ceph.spec index fd2794f74..7946fad10 100644 --- a/ceph/ceph.spec +++ b/ceph/ceph.spec @@ -43,26 +43,21 @@ %global _remote_tarball_prefix https://download.ceph.com/tarballs/ %endif %if 0%{?suse_version} -%bcond_with selinux -%bcond_with cephfs_java %bcond_with amqp_endpoint +%bcond_with cephfs_java %bcond_with kafka_endpoint -#Compat macro for new _fillupdir macro introduced in Nov 2017 -%if ! %{defined _fillupdir} -%global _fillupdir /var/adm/fillup-templates -%endif -%if 0%{?is_opensuse} -%bcond_without libradosstriper -%bcond_without ocf -%else %bcond_with libradosstriper -%bcond_with ocf -%endif %ifarch x86_64 aarch64 ppc64le %bcond_without lttng %else %bcond_with lttng %endif +%bcond_with ocf +%bcond_with selinux +#Compat macro for _fillupdir macro introduced in Nov 2017 +%if ! %{defined _fillupdir} +%global _fillupdir /var/adm/fillup-templates +%endif %endif %bcond_with seastar %if 0%{?fedora} || 0%{?suse_version} >= 1500 @@ -103,7 +98,7 @@ # main package definition ################################################################################# Name: ceph -Version: 15.2.1 +Version: 15.2.2 Release: 0%{?dist} %if 0%{?fedora} || 0%{?rhel} Epoch: 2 @@ -119,7 +114,7 @@ License: LGPL-2.1 and LGPL-3.0 and CC-BY-SA-3.0 and GPL-2.0 and BSL-1.0 and BSD- Group: System/Filesystems %endif URL: http://ceph.com/ -Source0: %{?_remote_tarball_prefix}ceph-15.2.1.tar.bz2 +Source0: %{?_remote_tarball_prefix}ceph-15.2.2.tar.bz2 %if 0%{?suse_version} # _insert_obs_source_lines_here ExclusiveArch: x86_64 aarch64 ppc64le s390x @@ -526,12 +521,16 @@ Requires: python%{python3_pkgversion}-cherrypy Requires: python%{python3_pkgversion}-jwt Requires: python%{python3_pkgversion}-routes Requires: python%{python3_pkgversion}-werkzeug +%if 0%{?weak_deps} +Recommends: python%{python3_pkgversion}-saml +%endif %endif %if 0%{?suse_version} Requires: python%{python3_pkgversion}-CherryPy Requires: python%{python3_pkgversion}-PyJWT Requires: python%{python3_pkgversion}-Routes Requires: python%{python3_pkgversion}-Werkzeug +Recommends: python%{python3_pkgversion}-python3-saml %endif %description mgr-dashboard ceph-mgr-dashboard is a manager module, providing a web-based application @@ -936,7 +935,7 @@ Summary: Ceph distributed file system client library %if 0%{?suse_version} Group: System/Libraries %endif -Obsoletes: libcephfs1 +Obsoletes: libcephfs1 < %{_epoch_prefix}%{version}-%{release} %if 0%{?rhel} || 0%{?fedora} Obsoletes: ceph-libs < %{_epoch_prefix}%{version}-%{release} Obsoletes: ceph-libcephfs @@ -1118,7 +1117,7 @@ This package provides Ceph’s default alerts for Prometheus. # common ################################################################################# %prep -%autosetup -p1 -n ceph-15.2.1 +%autosetup -p1 -n ceph-15.2.2 %build # LTO can be enabled as soon as the following GCC bug is fixed: @@ -1450,6 +1449,7 @@ exit 0 %files -n cephadm %{_sbindir}/cephadm +%{_mandir}/man8/cephadm.8* %{_sysconfdir}/sudoers.d/cephadm %attr(0700,cephadm,cephadm) %dir %{_sharedstatedir}/cephadm %attr(0700,cephadm,cephadm) %dir %{_sharedstatedir}/cephadm/.ssh @@ -2325,8 +2325,7 @@ if [ $1 -eq 0 ]; then fi fi exit 0 - -%endif # with selinux +%endif %files grafana-dashboards %if 0%{?suse_version} diff --git a/ceph/ceph.spec.in b/ceph/ceph.spec.in index 01e9ca5b7..2d20cd2bc 100644 --- a/ceph/ceph.spec.in +++ b/ceph/ceph.spec.in @@ -43,26 +43,21 @@ %global _remote_tarball_prefix https://download.ceph.com/tarballs/ %endif %if 0%{?suse_version} -%bcond_with selinux -%bcond_with cephfs_java %bcond_with amqp_endpoint +%bcond_with cephfs_java %bcond_with kafka_endpoint -#Compat macro for new _fillupdir macro introduced in Nov 2017 -%if ! %{defined _fillupdir} -%global _fillupdir /var/adm/fillup-templates -%endif -%if 0%{?is_opensuse} -%bcond_without libradosstriper -%bcond_without ocf -%else %bcond_with libradosstriper -%bcond_with ocf -%endif %ifarch x86_64 aarch64 ppc64le %bcond_without lttng %else %bcond_with lttng %endif +%bcond_with ocf +%bcond_with selinux +#Compat macro for _fillupdir macro introduced in Nov 2017 +%if ! %{defined _fillupdir} +%global _fillupdir /var/adm/fillup-templates +%endif %endif %bcond_with seastar %if 0%{?fedora} || 0%{?suse_version} >= 1500 @@ -526,12 +521,16 @@ Requires: python%{python3_pkgversion}-cherrypy Requires: python%{python3_pkgversion}-jwt Requires: python%{python3_pkgversion}-routes Requires: python%{python3_pkgversion}-werkzeug +%if 0%{?weak_deps} +Recommends: python%{python3_pkgversion}-saml +%endif %endif %if 0%{?suse_version} Requires: python%{python3_pkgversion}-CherryPy Requires: python%{python3_pkgversion}-PyJWT Requires: python%{python3_pkgversion}-Routes Requires: python%{python3_pkgversion}-Werkzeug +Recommends: python%{python3_pkgversion}-python3-saml %endif %description mgr-dashboard ceph-mgr-dashboard is a manager module, providing a web-based application @@ -936,7 +935,7 @@ Summary: Ceph distributed file system client library %if 0%{?suse_version} Group: System/Libraries %endif -Obsoletes: libcephfs1 +Obsoletes: libcephfs1 < %{_epoch_prefix}%{version}-%{release} %if 0%{?rhel} || 0%{?fedora} Obsoletes: ceph-libs < %{_epoch_prefix}%{version}-%{release} Obsoletes: ceph-libcephfs @@ -1450,6 +1449,7 @@ exit 0 %files -n cephadm %{_sbindir}/cephadm +%{_mandir}/man8/cephadm.8* %{_sysconfdir}/sudoers.d/cephadm %attr(0700,cephadm,cephadm) %dir %{_sharedstatedir}/cephadm %attr(0700,cephadm,cephadm) %dir %{_sharedstatedir}/cephadm/.ssh @@ -2325,8 +2325,7 @@ if [ $1 -eq 0 ]; then fi fi exit 0 - -%endif # with selinux +%endif %files grafana-dashboards %if 0%{?suse_version} diff --git a/ceph/changelog.upstream b/ceph/changelog.upstream index 2bcd6d679..b11435736 100644 --- a/ceph/changelog.upstream +++ b/ceph/changelog.upstream @@ -1,7 +1,13 @@ -ceph (15.2.1-1bionic) bionic; urgency=medium +ceph (15.2.2-1bionic) bionic; urgency=medium - -- Jenkins Build Slave User Wed, 08 Apr 2020 18:04:14 +0000 + -- Jenkins Build Slave User Mon, 18 May 2020 16:37:36 +0000 + +ceph (15.2.2-1) stable; urgency=medium + + * New upstream release + + -- Ceph Release Team Mon, 18 May 2020 16:25:10 +0000 ceph (15.2.1-1) stable; urgency=medium diff --git a/ceph/cmake/modules/FindFUSE.cmake b/ceph/cmake/modules/FindFUSE.cmake new file mode 100644 index 000000000..1d0766f33 --- /dev/null +++ b/ceph/cmake/modules/FindFUSE.cmake @@ -0,0 +1,58 @@ +# This module can find FUSE Library +# +# The following variables will be defined for your use: +# - FUSE_FOUND : was FUSE found? +# - FUSE_INCLUDE_DIRS : FUSE include directory +# - FUSE_LIBRARIES : FUSE library +# - FUSE_VERSION : the version of the FUSE library found + +if(PACKAGE_FIND_VERSION AND PACKAGE_FIND_VERSION VERSION_LESS "3.0") + set(fuse_names fuse) + set(fuse_suffixes fuse) +else() + set(fuse_names fuse3 fuse) + set(fuse_suffixes fuse3 fuse) +endif() + +if(APPLE) + list(APPEND fuse_names libosxfuse.dylib) + list(APPEND fuse_suffixes osxfuse) +endif() + +find_path( + FUSE_INCLUDE_DIR + NAMES fuse_common.h fuse_lowlevel.h fuse.h + PATH_SUFFIXES ${fuse_suffixes}) + +find_library(FUSE_LIBRARIES + NAMES ${fuse_names} + PATHS /usr/local/lib64 /usr/local/lib) + +foreach(ver "MAJOR" "MINOR") + file(STRINGS "${FUSE_INCLUDE_DIR}/fuse_common.h" fuse_ver_${ver}_line + REGEX "^#define[\t ]+FUSE_${ver}_VERSION[\t ]+[0-9]+$") + string(REGEX REPLACE ".*#define[\t ]+FUSE_${ver}_VERSION[\t ]+([0-9]+)$" + "\\1" FUSE_VERSION_${ver} "${fuse_ver_${ver}_line}") +endforeach() +set(FUSE_VERSION + "${FUSE_VERSION_MAJOR}.${FUSE_VERSION_MINOR}") + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(FUSE + REQUIRED_VARS FUSE_LIBRARIES FUSE_INCLUDE_DIR + VERSION_VAR FUSE_VERSION) + +mark_as_advanced( + FUSE_INCLUDE_DIR) + +if(FUSE_FOUND) + set(FUSE_INCLUDE_DIRS ${FUSE_INCLUDE_DIR}) + if(NOT TARGET FUSE::FUSE) + add_library(FUSE::FUSE UNKNOWN IMPORTED) + set_target_properties(FUSE::FUSE PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${FUSE_INCLUDE_DIRS}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${FUSE_LIBRARIES}" + VERSION "${FUSE_VERSION}") + endif() +endif() diff --git a/ceph/cmake/modules/Findfuse.cmake b/ceph/cmake/modules/Findfuse.cmake deleted file mode 100644 index e7a7ff08d..000000000 --- a/ceph/cmake/modules/Findfuse.cmake +++ /dev/null @@ -1,28 +0,0 @@ -# This module can find FUSE Library -# -# The following variables will be defined for your use: -# - FUSE_FOUND : was FUSE found? -# - FUSE_INCLUDE_DIRS : FUSE include directory -# - FUSE_LIBRARIES : FUSE library - -find_path( - FUSE_INCLUDE_DIRS - NAMES fuse_common.h fuse_lowlevel.h fuse.h - PATHS /usr/local/include/osxfuse /usr/local/include - PATH_SUFFIXES fuse) - -set(fuse_names fuse) -if(APPLE) - list(APPEND fuse_names libosxfuse.dylib) -endif() - -find_library(FUSE_LIBRARIES - NAMES ${fuse_names} - PATHS /usr/local/lib64 /usr/local/lib) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(fuse DEFAULT_MSG - FUSE_INCLUDE_DIRS FUSE_LIBRARIES) - -mark_as_advanced( - FUSE_INCLUDE_DIRS FUSE_LIBRARIES) diff --git a/ceph/debian/cephadm.install b/ceph/debian/cephadm.install index 262f3eec6..49d6cc34d 100644 --- a/ceph/debian/cephadm.install +++ b/ceph/debian/cephadm.install @@ -1,2 +1,3 @@ usr/sbin/cephadm +usr/share/man/man8/cephadm.8 etc/sudoers.d/cephadm diff --git a/ceph/doc/cephadm/client-setup.rst b/ceph/doc/cephadm/client-setup.rst new file mode 100644 index 000000000..dd0bc3285 --- /dev/null +++ b/ceph/doc/cephadm/client-setup.rst @@ -0,0 +1,36 @@ +======================= +Basic Ceph Client Setup +======================= +Client machines need some basic configuration in order to interact with +a cluster. This document describes how to configure a client machine +for cluster interaction. + +.. note:: Most client machines only need the `ceph-common` package and + its dependencies installed. That will supply the basic `ceph` + and `rados` commands, as well as other commands like + `mount.ceph` and `rbd`. + +Config File Setup +================= +Client machines can generally get away with a smaller config file than +a full-fledged cluster member. To generate a minimal config file, log +into a host that is already configured as a client or running a cluster +daemon, and then run:: + + ceph config generate-minimal-conf + +This will generate a minimal config file that will tell the client how to +reach the Ceph Monitors. The contents of this file should typically be +installed in `/etc/ceph/ceph.conf`. + +Keyring Setup +============= +Most Ceph clusters are run with authentication enabled, and the client will +need keys in order to communicate with cluster machines. To generate a +keyring file with credentials for `client.fs`, log into an extant cluster +member and run:: + + ceph auth get-or-create client.fs + +The resulting output should be put into a keyring file, typically +`/etc/ceph/ceph.keyring`. diff --git a/ceph/doc/cephadm/drivegroups.rst b/ceph/doc/cephadm/drivegroups.rst index 181ff3ef8..ae9b9205f 100644 --- a/ceph/doc/cephadm/drivegroups.rst +++ b/ceph/doc/cephadm/drivegroups.rst @@ -173,7 +173,7 @@ This example would deploy all OSDs with encryption enabled. host_pattern: '*' data_devices: all: true - encrypted: true + encrypted: true See a full list in the DriveGroupSpecs diff --git a/ceph/doc/cephadm/index.rst b/ceph/doc/cephadm/index.rst index e1be17e8d..4ffcab80a 100644 --- a/ceph/doc/cephadm/index.rst +++ b/ceph/doc/cephadm/index.rst @@ -34,5 +34,6 @@ versions of Ceph. Cephadm operations Cephadm monitoring Cephadm CLI <../mgr/orchestrator> + Client Setup DriveGroups troubleshooting diff --git a/ceph/doc/cephadm/install.rst b/ceph/doc/cephadm/install.rst index 65ac09d8c..ca4ff970d 100644 --- a/ceph/doc/cephadm/install.rst +++ b/ceph/doc/cephadm/install.rst @@ -302,7 +302,9 @@ see :ref:`fs-volumes-and-subvolumes`. To deploy metadata servers:: - # ceph orch apply mds ** ** [** ...] + # ceph orch apply mds ** --placement="** [** ...]" + +See :ref:`orchestrator-cli-placement-spec` for details of the placement specification. Deploy RGWs =========== @@ -332,7 +334,7 @@ Next create a zone:: To deploy a set of radosgw daemons for a particular realm and zone:: - # ceph orch apply rgw ** ** ** [** ...] + # ceph orch apply rgw ** ** --placement="** [** ...]" For example, to deploy 2 rgw daemons serving the *myorg* realm and the *us-east-1* zone on *myhost1* and *myhost2*:: @@ -340,7 +342,9 @@ zone on *myhost1* and *myhost2*:: # radosgw-admin realm create --rgw-realm=myorg --default # radosgw-admin zonegroup create --rgw-zonegroup=default --master --default # radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=us-east-1 --master --default - # ceph orch apply rgw myorg us-east-1 2 myhost1 myhost2 + # ceph orch apply rgw myorg us-east-1 --placement="2 myhost1 myhost2" + +See :ref:`orchestrator-cli-placement-spec` for details of the placement specification. Deploying NFS ganesha ===================== @@ -350,9 +354,12 @@ and optional *namespace* To deploy a NFS Ganesha gateway,:: - # ceph orch apply nfs ** ** ** ** [** ...] + # ceph orch apply nfs ** ** ** --placement="** [** ...]" For example, to deploy NFS with a service id of *foo*, that will use the RADOS pool *nfs-ganesha* and namespace *nfs-ns*,:: # ceph orch apply nfs foo nfs-ganesha nfs-ns + +See :ref:`orchestrator-cli-placement-spec` for details of the placement specification. + diff --git a/ceph/doc/cephadm/troubleshooting.rst b/ceph/doc/cephadm/troubleshooting.rst index 93f42025f..da4f55317 100644 --- a/ceph/doc/cephadm/troubleshooting.rst +++ b/ceph/doc/cephadm/troubleshooting.rst @@ -102,3 +102,49 @@ Manually running containers Cephadm writes small wrappers that run a containers. Refer to ``/var/lib/ceph///unit.run`` for the container execution command. + + +ssh errors +---------- + +Error message:: + + xxxxxx.gateway_bootstrap.HostNotFound: -F /tmp/cephadm-conf-kbqvkrkw root@10.10.1.2 + raise OrchestratorError('Failed to connect to %s (%s). Check that the host is reachable and accepts connections using the cephadm SSH key' % (host, addr)) from + orchestrator._interface.OrchestratorError: Failed to connect to 10.10.1.2 (10.10.1.2). Check that the host is reachable and accepts connections using the cephadm SSH key + +Things users can do: + +1. Ensure cephadm has an SSH identity key:: + + [root@mon1~]# cephadm shell -- ceph config-key get mgr/cephadm/ssh_identity_key > key + INFO:cephadm:Inferring fsid f8edc08a-7f17-11ea-8707-000c2915dd98 + INFO:cephadm:Using recent ceph image docker.io/ceph/ceph:v15 obtained 'mgr/cephadm/ssh_identity_key' + [root@mon1 ~] # chmod 0600 key + + If this fails, cephadm doesn't have a key. Fix this by running the following command:: + + [root@mon1 ~]# cephadm shell -- ceph cephadm generate-ssh-key + + or:: + + [root@mon1 ~]# cat key | cephadm shell -- ceph cephadm set-ssk-key -i - + +2. Ensure that the ssh config is correct:: + + [root@mon1 ~]# cephadm shell -- ceph cephadm get-ssh-config > config + +3. Verify that we can connect to the host:: + + [root@mon1 ~]# ssh -F config -i key root@mon1 + +4. There is a limitation right now: the ssh user is always `root`. + + + +Verifying that the Public Key is Listed in the authorized_keys file +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +To verify that the public key is in the authorized_keys file, run the following commands:: + + [root@mon1 ~]# cephadm shell -- ceph config-key get mgr/cephadm/ssh_identity_pub > key.pub + [root@mon1 ~]# grep "`cat key.pub`" /root/.ssh/authorized_keys diff --git a/ceph/doc/cephadm/upgrade.rst b/ceph/doc/cephadm/upgrade.rst index ba194f706..64ba6dadb 100644 --- a/ceph/doc/cephadm/upgrade.rst +++ b/ceph/doc/cephadm/upgrade.rst @@ -28,11 +28,11 @@ and your cluster is healthy. To upgrade (or downgrade) to a specific release:: - # ceph orch upgrade start --version + # ceph orch upgrade start --ceph-version For example, to upgrade to v15.2.1:: - # ceph orch upgrade start --version 15.2.1 + # ceph orch upgrade start --ceph-version 15.2.1 Monitoring the upgrade @@ -99,7 +99,7 @@ one or more hosts in the cluster. You can cancel the existing upgrade and specify a different target version with:: # ceph orch upgrade stop - # ceph orch upgrade start --version + # ceph orch upgrade start --ceph-version Using customized container images diff --git a/ceph/doc/cephfs/fs-volumes.rst b/ceph/doc/cephfs/fs-volumes.rst index 62eb4ec0e..f602cc106 100644 --- a/ceph/doc/cephfs/fs-volumes.rst +++ b/ceph/doc/cephfs/fs-volumes.rst @@ -152,6 +152,27 @@ Fetch the absolute path of a subvolume using:: $ ceph fs subvolume getpath [--group_name ] +Fetch the metadata of a subvolume using:: + + $ ceph fs subvolume info [--group_name ] + +The output format is json and contains fields as follows. + +* atime: access time of subvolume path in the format "YYYY-MM-DD HH:MM:SS" +* mtime: modification time of subvolume path in the format "YYYY-MM-DD HH:MM:SS" +* ctime: change time of subvolume path in the format "YYYY-MM-DD HH:MM:SS" +* uid: uid of subvolume path +* gid: gid of subvolume path +* mode: mode of subvolume path +* mon_addrs: list of monitor addresses +* bytes_pcent: quota used in percentage if quota is set, else displays "undefined" +* bytes_quota: quota size in bytes if quota is set, else displays "infinite" +* bytes_used: current used size of the subvolume in bytes +* created_at: time of creation of subvolume in the format "YYYY-MM-DD HH:MM:SS" +* data_pool: data pool the subvolume belongs to +* path: absolute path of a subvolume +* type: subvolume type indicating whether it's clone or subvolume + List subvolumes using:: $ ceph fs subvolume ls [--group_name ] diff --git a/ceph/doc/dev/cephadm.rst b/ceph/doc/dev/cephadm.rst new file mode 100644 index 000000000..2fd475f13 --- /dev/null +++ b/ceph/doc/dev/cephadm.rst @@ -0,0 +1,87 @@ +======================= +Developing with cephadm +======================= + +There are several ways to develop with cephadm. Which you use depends +on what you're trying to accomplish. + +vstart --cephadm +================ + +- Start a cluster with vstart, with cephadm configured +- Manage any additional daemons with cephadm + +In this case, the mon and manager at a minimum are running in the usual +vstart way, not managed by cephadm. But cephadm is enabled and the local +host is added, so you can deploy additional daemons or add additional hosts. + +This works well for developing cephadm itself, because any mgr/cephadm +or cephadm/cephadm code changes can be applied by kicking ceph-mgr +with ``ceph mgr fail x``. (When the mgr (re)starts, it loads the +cephadm/cephadm script into memory.) + +:: + + MON=1 MGR=1 OSD=0 MDS=0 ../src/vstart.sh -d -n -x --cephadm + +- ``~/.ssh/id_dsa[.pub]`` is used as the cluster key. It is assumed that + this key is authorized to ssh to root@`hostname`. +- No service spec is defined for mon or mgr, which means that cephadm + does not try to manage them. +- You'll see health warnings from cephadm about stray daemons--that's because + the vstart-launched mon and mgr aren't controlled by cephadm. +- The default image is ``quay.io/ceph-ci/ceph:master``, but you can change + this by passing ``-o container_image=...`` or ``ceph config set global container_image ...``. + + +cstart and cpatch +================= + +The ``cstart.sh`` script will launch a cluster using cephadm and put the +conf and keyring in your build dir, so that the ``bin/ceph ...`` CLI works +(just like with vstart). The ``ckill.sh`` script will tear it down. + +- A unique but stable fsid is stored in ``fsid`` (in the build dir). +- The mon port is random, just like with vstart. +- The container image is ``quay.io/ceph-ci/ceph:$tag`` where $tag is + the first 8 chars of the fsid. +- If the container image doesn't exist yet when you run cstart for the + first time, it is built with cpatch. + +There are a few advantages here: + +- The cluster is a "normal" cephadm cluster that looks and behaves + just like a user's cluster would. In contract, vstart and teuthology + clusters tend to be special in subtle (and not-so-subtle) ways. + +To start a test cluster:: + + sudo ../src/cstart.sh + +The last line of this will be a line you can cut+paste to update the +container image. For instance:: + + sudo ../src/script/cpach -t quay.io/ceph-ci/ceph:8f509f4e + +By default, cpatch will patch everything it can think of from the local +build dir into the container image. If you are working on a specific +part of the system, though, can you get away with smaller changes so that +cpatch runs faster. For instance:: + + sudo ../src/script/cpach -t quay.io/ceph-ci/ceph:8f509f4e --py + +will update the mgr modules (minus the dashboard). Or:: + + sudo ../src/script/cpach -t quay.io/ceph-ci/ceph:8f509f4e --core + +will do most binaries and libraries. Pass ``-h`` to cpatch for all options. + +Once the container is updated, you can refresh/restart daemons by bouncing +them with:: + + sudo systemctl restart ceph-`cat fsid`.target + +When you're done, you can tear down the cluster with:: + + sudo ../src/ckill.sh # or, + sudo ../src/cephadm/cephadm rm-cluster --force --fsid `cat fsid` diff --git a/ceph/doc/install/index.rst b/ceph/doc/install/index.rst index 507247ca9..df4390c58 100644 --- a/ceph/doc/install/index.rst +++ b/ceph/doc/install/index.rst @@ -13,7 +13,7 @@ Recommended methods :ref:`Cephadm ` installs and manages a Ceph cluster using containers and systemd, with tight integration with the CLI and dashboard GUI. -* cephadm only supports only Octopus and newer releases. +* cephadm only supports Octopus and newer releases. * cephadm is fully integrated with the new orchestration API and fully supports the new CLI and dashboard features to manage cluster deployment. diff --git a/ceph/doc/man/8/CMakeLists.txt b/ceph/doc/man/8/CMakeLists.txt index 1fb6465ae..02655a8cd 100644 --- a/ceph/doc/man/8/CMakeLists.txt +++ b/ceph/doc/man/8/CMakeLists.txt @@ -37,7 +37,8 @@ list(APPEND man_srcs ${osd_srcs} ${mon_srcs} ceph-mds.rst - librados-config.rst) + librados-config.rst + cephadm.rst) if(HAVE_LIBFUSE) list(APPEND man_srcs diff --git a/ceph/doc/man/8/cephadm.rst b/ceph/doc/man/8/cephadm.rst new file mode 100644 index 000000000..9c24d5e82 --- /dev/null +++ b/ceph/doc/man/8/cephadm.rst @@ -0,0 +1,441 @@ +:orphan: + +========================================= + cephadm -- manage the local cephadm host +========================================= + +.. program:: cephadm + +Synopsis +======== + +| **cephadm**** [-h] [--image IMAGE] [--docker] [--data-dir DATA_DIR] +| [--log-dir LOG_DIR] [--logrotate-dir LOGROTATE_DIR] +| [--unit-dir UNIT_DIR] [--verbose] [--timeout TIMEOUT] +| [--retry RETRY] +| {version,pull,inspect-image,ls,list-networks,adopt,rm-daemon,rm-cluster,run,shell,enter,ceph-volume,unit,logs,bootstrap,deploy,check-host,prepare-host,add-repo,rm-repo,install} +| ... + + +| **cephadm** **pull** + +| **cephadm** **inspect-image** + +| **cephadm** **ls** + +| **cephadm** **list-networks** + +| **cephadm** **adopt** [-h] --name NAME --style STYLE [--cluster CLUSTER] +| [--legacy-dir LEGACY_DIR] [--config-json CONFIG_JSON] +| [--skip-firewalld] [--skip-pull] + +| **cephadm** **rm-daemon** [-h] --name NAME --fsid FSID [--force] +| [--force-delete-data] + +| **cephadm** **rm-cluster** [-h] --fsid FSID [--force] + +| **cephadm** **run** [-h] --name NAME --fsid FSID + +| **cephadm** **shell** [-h] [--fsid FSID] [--name NAME] [--config CONFIG] + [--keyring KEYRING] [--env ENV] + [--] [command [command ...]] + +| **cephadm** **enter** [-h] [--fsid FSID] --name NAME [command [command ...]] + +| **cephadm** **ceph-volume** [-h] [--fsid FSID] [--config-json CONFIG_JSON] + [--config CONFIG] [--keyring KEYRING] + command [command ...] + +| **cephadm** **logs** [-h] [--fsid FSID] --name NAME [command [command ...]] + +| **cephadm** **bootstrap** [-h] [--config CONFIG] [--mon-id MON_ID] +| [--mon-addrv MON_ADDRV] [--mon-ip MON_IP] +| [--mgr-id MGR_ID] [--fsid FSID] +| [--output-dir OUTPUT_DIR] +| [--output-keyring OUTPUT_KEYRING] +| [--output-config OUTPUT_CONFIG] +| [--output-pub-ssh-key OUTPUT_PUB_SSH_KEY] +| [--skip-ssh] +| [--initial-dashboard-user INITIAL_DASHBOARD_USER] +| [--initial-dashboard-password INITIAL_DASHBOARD_PASSWORD] +| [--dashboard-key DASHBOARD_KEY] +| [--dashboard-crt DASHBOARD_CRT] [--skip-mon-network] +| [--skip-dashboard] [--dashboard-password-noupdate] +| [--no-minimize-config] [--skip-ping-check] +| [--skip-pull] [--skip-firewalld] [--allow-overwrite] +| [--allow-fqdn-hostname] [--skip-prepare-host] +| [--orphan-initial-daemons] [--skip-monitoring-stack] + + +| **cephadm** **deploy** [-h] --name NAME --fsid FSID [--config CONFIG] +| [--config-json CONFIG_JSON] [--keyring KEYRING] +| [--key KEY] [--osd-fsid OSD_FSID] [--skip-firewalld] +| [--reconfig] [--allow-ptrace] + +| **cephadm** **check-host** [-h] [--expect-hostname EXPECT_HOSTNAME] + +| **cephadm** **prepare-host** + +| **cephadm** **add-repo** [-h] [--release RELEASE] [--version VERSION] +| [--dev DEV] [--dev-commit DEV_COMMIT] +| [--gpg-url GPG_URL] [--repo-url REPO_URL] + + +| **cephadm** **rm-repo** + +| **cephadm** **install** [-h] [packages [packages ...]] + + + + +Description +=========== + +:program:`cephadm` is a command line tool to manage the local host for the cephadm orchestrator. + +It provides commands to investigate and modify the state of the current host. + +:program:`cephadm` is not required on all hosts, but useful when investigating a particular +daemon. + +Options +======= + +.. option:: --image IMAGE + + container image. Can also be set via the + "CEPHADM_IMAGE" env var (default: None) + +.. option:: --docker + + use docker instead of podman (default: False) +.. option::data-dir DATA_DIR + + base directory for daemon data (default:/var/lib/ceph) + +.. option:: --log-dir LOG_DIR + + base directory for daemon logs (default:.. option:: /var/log/ceph) + +.. option:: --logrotate-dir LOGROTATE_DIR + + location of logrotate configuration files (default: /etc/logrotate.d) + +.. option:: --unit-dir UNIT_DIR + + base directory for systemd units (default: /etc/systemd/system) + +.. option:: --verbose, -v + + Show debug-level log messages (default: False) + +.. option:: --timeout TIMEOUT + + timeout in seconds (default: None) + +.. option:: --retry RETRY + + max number of retries (default: 10) + + +Commands +======== + +add-repo +-------- + +configure local package repository to also include the ceph repository. + +Arguments: + +* [--release RELEASE] use latest version of a named release (e.g., octopus) +* [--version VERSION] use specific upstream version (x.y.z) +* [--dev DEV] use specified bleeding edge build from git branch or tag +* [--dev-commit DEV_COMMIT] use specified bleeding edge build from git commit +* [--gpg-url GPG_URL] specify alternative GPG key location +* [--repo-url REPO_URL] specify alternative repo location + + +adopt +----- + +Adopt a daemon deployed with a different deployment tool. + +Arguments: + +* [--name NAME, -n NAME] daemon name (type.id) +* [--style STYLE] deployment style (legacy, ...) +* [--cluster CLUSTER] cluster name +* [--legacy-dir LEGACY_DIR] base directory for legacy daemon data +* [--config-json CONFIG_JSON] Additional configuration information in JSON format +* [--skip-firewalld] Do not configure firewalld +* [--skip-pull] do not pull the latest image before adopting + + +bootstrap +--------- + +Bootstrap a cluster on the local host. It deploys a MON and a MGR and then also automatically +deploys the monitoring stack on this host (see --skip-monitoring-stack) and calls +``ceph orch host add $(hostname)`` (see --skip-ssh). + +Arguments: + +* [--config CONFIG, -c CONFIG] ceph conf file to incorporate +* [--mon-id MON_ID] mon id (default: local hostname) +* [--mon-addrv MON_ADDRV] mon IPs (e.g., [v2:localipaddr:3300,v1:localipaddr:6789]) +* [--mon-ip MON_IP] mon IP +* [--mgr-id MGR_ID] mgr id (default: randomly generated) +* [--fsid FSID] cluster FSID +* [--output-dir OUTPUT_DIR] directory to write config, keyring, and pub key files +* [--output-keyring OUTPUT_KEYRING] location to write keyring file with new cluster admin and mon keys +* [--output-config OUTPUT_CONFIG] location to write conf file to connect to new cluster +* [--output-pub-ssh-key OUTPUT_PUB_SSH_KEY] location to write the cluster's public SSH key +* [--skip-ssh skip setup of ssh key on local host +* [--initial-dashboard-user INITIAL_DASHBOARD_USER] Initial user for the dashboard +* [--initial-dashboard-password INITIAL_DASHBOARD_PASSWORD] Initial password for the initial dashboard user +* [--dashboard-key DASHBOARD_KEY] Dashboard key +* [--dashboard-crt DASHBOARD_CRT] Dashboard certificate +* [--skip-mon-network] set mon public_network based on bootstrap mon ip +* [--skip-dashboard] do not enable the Ceph Dashboard +* [--dashboard-password-noupdate] stop forced dashboard password change +* [--no-minimize-config] do not assimilate and minimize the config file +* [--skip-ping-check] do not verify that mon IP is pingable +* [--skip-pull] do not pull the latest image before bootstrapping +* [--skip-firewalld] Do not configure firewalld +* [--allow-overwrite] allow overwrite of existing --output-* config/keyring/ssh files +* [--allow-fqdn-hostname] allow hostname that is fully-qualified (contains ".") +* [--skip-prepare-host] Do not prepare host +* [--orphan-initial-daemons] Do not create initial mon, mgr, and crash service specs +* [--skip-monitoring-stack] Do not automatically provision monitoring stack] (prometheus, grafana, alertmanager, node-exporter) + + +ceph-volume +----------- + +Run ceph-volume inside a container:: + + cephadm ceph-volume inventory + +Positional arguments: +* [command] command + +Arguments: + +* [--fsid FSID] cluster FSID +* [--config-json CONFIG_JSON] JSON file with config and (client.bootrap-osd) key +* [--config CONFIG, -c CONFIG] ceph conf file +* [--keyring KEYRING, -k KEYRING] ceph.keyring to pass through to the container + + +check-host +---------- + +check host configuration to be suitable for a Ceph cluster. + +Arguments: + +* [--expect-hostname EXPECT_HOSTNAME] Check that hostname matches an expected value + + +deploy +------ + +deploy a daemon on the local host. Used by the orchestrator CLI:: + + cephadm shell -- ceph orch apply ... + +Arguments: + +* [--name NAME] daemon name (type.id) +* [--fsid FSID] cluster FSID +* [--config CONFIG, -c CONFIG] config file for new daemon +* [--config-json CONFIG_JSON] Additional configuration information in JSON format +* [--keyring KEYRING] keyring for new daemon +* [--key KEY] key for new daemon +* [--osd-fsid OSD_FSID] OSD uuid, if creating an OSD container +* [--skip-firewalld] Do not configure firewalld +* [--reconfig] Reconfigure a previously deployed daemon +* [--allow-ptrace] Allow SYS_PTRACE on daemon container + + +enter +----- + +Run an interactive shell inside a running daemon container:: + + cephadm enter --name mgr.myhost.ysubfo + +Positional arguments: +* [command] command + +Arguments: + +* [--fsid FSID] cluster FSID +* [--name NAME, -n NAME] daemon name (type.id) + +install +------- + +install ceph package(s) + +Positional arguments: + +* [packages] packages + + +inspect-image +------------- + +inspect local ceph container image. + +list-networks +------------- + +list IP networks + + +ls +-- + +list daemon instances known to cephadm on **this** host:: + + $ cephadm ls + [ + { + "style": "cephadm:v1", + "name": "mgr.storage-14b-1.ysubfo", + "fsid": "5110cb22-8332-11ea-9148-0894ef7e8bdc", + "enabled": true, + "state": "running", + "container_id": "8562de72370a3836473ecfff8a22c9ccdd99815386b4692a2b30924fb5493c44", + "container_image_name": "docker.io/ceph/ceph:v15", + "container_image_id": "bc83a388465f0568dab4501fb7684398dca8b50ca12a342a57f21815721723c2", + "version": "15.2.1", + "started": "2020-04-21T01:16:41.831456", + "created": "2020-04-21T01:16:41.775024", + "deployed": "2020-04-21T01:16:41.415021", + "configured": "2020-04-21T01:16:41.775024" + }, + ... + +logs +---- + +print journald logs for a daemon container:: + + cephadm logs --name mgr.myhost.ysubfo + +This is similar to:: + + journalctl -u mgr.myhost.ysubfo + + +prepare-host +------------ + +prepare a host for cephadm use + +Arguments: + +* [--expect-hostname EXPECT_HOSTNAME] Set hostname + + +pull +---- + +Pull the ceph image:: + + cephadm pull + + +rm-daemon +--------- + +Remove a specific daemon instance + +Arguments: + +* [--name NAME, -n NAME] daemon name (type.id) +* [--fsid FSID] cluster FSID +* [--force] proceed, even though this may destroy valuable data +* [--force-delete-data] delete valuable daemon data instead of making a backup + + +rm-cluster +---------- + +remove all daemons for a cluster + +Arguments: + +* [--fsid FSID] cluster FSID +* [--force] proceed, even though this may destroy valuable data + +rm-repo +------- + +remove package repository configuration + +run +--- + +run a ceph daemon, in a container, in the foreground + +Arguments: + +* [--name NAME, -n NAME] daemon name (type.id) +* [--fsid FSID] cluster FSID + + +shell +----- + +Run an interactive shell:: + + cephadm shell + +Or one specific command inside a container:: + + cephadm shell -- ceph orch ls + + +Positional arguments: + +* [command] command (optional) + +Arguments: + +* [--fsid FSID] cluster FSID +* [--name NAME, -n NAME] daemon name (type.id) +* [--config CONFIG, -c CONFIG] ceph.conf to pass through to the container +* [--keyring KEYRING, -k KEYRING] ceph.keyring to pass through to the container +* [--env ENV, -e ENV] set environment variable + + +unit +---- + +Operate on the daemon's systemd unit. + +Positional arguments: + +* [command] systemd command (start, stop, restart, enable, disable, ...) + +Arguments: + +* [--fsid FSID] cluster FSID +* [--name NAME, -n NAME] daemon name (type.id) + + +Availability +============ + +:program:`cephadm` is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to +the documentation at http://docs.ceph.com/ for more information. + + +See also +======== + +:doc:`ceph-volume `\(8), diff --git a/ceph/doc/mgr/orchestrator.rst b/ceph/doc/mgr/orchestrator.rst index 47545f7f9..440961f1b 100644 --- a/ceph/doc/mgr/orchestrator.rst +++ b/ceph/doc/mgr/orchestrator.rst @@ -206,18 +206,29 @@ services of a particular type via optional --type parameter :: - ceph orch ps - ceph orch service ls [--host host] [--svc_type type] [--refresh] + ceph orch ls [--service_type type] [--service_name name] [--export] [--format f] [--refresh] Discover the status of a particular service or daemons:: - ceph orch service ls --svc_type type --svc_id [--refresh] + ceph orch ls --service_type type --service_name [--refresh] + +Export the service specs known to the orchestrator as yaml in format +that is compatible to ``ceph orch apply -i``:: + + ceph orch ls --export + +Daemon Status +============= +Print a list of all daemons known to the orchestrator:: + + ceph orch ps [--hostname host] [--daemon_type type] [--service_name name] [--daemon_id id] [--format f] [--refresh] + Query the status of a particular service instance (mon, osd, mds, rgw). For OSDs the id is the numeric OSD ID, for MDS services it is the file system name:: - ceph orch daemon status  [--refresh] + ceph orch ps --daemon_type osd --daemon_id 0 .. _orchestrator-cli-cephfs: diff --git a/ceph/doc/releases/nautilus.rst b/ceph/doc/releases/nautilus.rst index 986403aee..87fb0bba2 100644 --- a/ceph/doc/releases/nautilus.rst +++ b/ceph/doc/releases/nautilus.rst @@ -1667,6 +1667,9 @@ Instructions and verify that each monitor has both a ``v2:`` and ``v1:`` address listed. + Running nautilus OSDs will not bind to their v2 address automatically. + They must be restarted for that to happen. + .. important:: Before this step is run, the following command must already have been run: diff --git a/ceph/monitoring/grafana/dashboards/ceph-cluster.json b/ceph/monitoring/grafana/dashboards/ceph-cluster.json index 549bb1ea2..447dbb293 100644 --- a/ceph/monitoring/grafana/dashboards/ceph-cluster.json +++ b/ceph/monitoring/grafana/dashboards/ceph-cluster.json @@ -260,6 +260,7 @@ "#d44a3a" ], "datasource": "$datasource", + "decimals": 2, "format": "percentunit", "gauge": { "maxValue": 100, diff --git a/ceph/monitoring/grafana/dashboards/host-details.json b/ceph/monitoring/grafana/dashboards/host-details.json index 6a9604715..67e759d91 100644 --- a/ceph/monitoring/grafana/dashboards/host-details.json +++ b/ceph/monitoring/grafana/dashboards/host-details.json @@ -798,7 +798,7 @@ "steppedLine": false, "targets": [ { - "expr": "(irate(node_disk_writes_completed{instance=~\"($ceph_hosts).*\"}[5m]) or irate(node_disk_writes_completed_total{instance=~\"($ceph_hosts).*\"}[5m])) * on(instance, device) group_left(ceph_daemon) label_replace(label_replace(ceph_disk_occupation, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"(.*)\")", + "expr": "(irate(node_disk_writes_completed{instance=~\"($ceph_hosts).*\"}[5m]) or irate(node_disk_writes_completed_total{instance=~\"($ceph_hosts).*\"}[5m])) * on(instance, device) group_left(ceph_daemon) label_replace(label_replace(ceph_disk_occupation, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}}({{ceph_daemon}}) writes", @@ -807,7 +807,7 @@ "textEditor": true }, { - "expr": "(irate(node_disk_reads_completed{instance=~\"($ceph_hosts).*\"}[5m]) or irate(node_disk_reads_completed_total{instance=~\"($ceph_hosts).*\"}[5m])) * on(instance, device) group_left(ceph_daemon) label_replace(label_replace(ceph_disk_occupation, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"(.*)\")", + "expr": "(irate(node_disk_reads_completed{instance=~\"($ceph_hosts).*\"}[5m]) or irate(node_disk_reads_completed_total{instance=~\"($ceph_hosts).*\"}[5m])) * on(instance, device) group_left(ceph_daemon) label_replace(label_replace(ceph_disk_occupation, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -899,14 +899,14 @@ "steppedLine": false, "targets": [ { - "expr": "(irate(node_disk_bytes_written{instance=~\"($ceph_hosts).*\"}[5m]) or irate(node_disk_written_bytes_total{instance=~\"($ceph_hosts).*\"}[5m])) * on(instance, device) group_left(ceph_daemon) label_replace(label_replace(ceph_disk_occupation, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"(.*)\")", + "expr": "label_replace((irate(node_disk_bytes_written{instance=~\"($ceph_hosts).*\"}[5m]) or irate(node_disk_written_bytes_total{instance=~\"($ceph_hosts).*\"}[5m])), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") * on(instance, device) group_left(ceph_daemon) label_replace(label_replace(ceph_disk_occupation, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}}({{ceph_daemon}}) write", "refId": "B" }, { - "expr": "(irate(node_disk_bytes_read{instance=~\"($ceph_hosts).*\"}[5m]) or irate(node_disk_read_bytes_total{instance=~\"($ceph_hosts).*\"}[5m])) * on(instance, device) group_left(ceph_daemon) label_replace(label_replace(ceph_disk_occupation, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"(.*)\")", + "expr": "label_replace((irate(node_disk_bytes_read{instance=~\"($ceph_hosts).*\"}[5m]) or irate(node_disk_read_bytes_total{instance=~\"($ceph_hosts).*\"}[5m])), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") * on(instance, device) group_left(ceph_daemon) label_replace(label_replace(ceph_disk_occupation, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}}({{ceph_daemon}}) read", @@ -992,7 +992,7 @@ "steppedLine": false, "targets": [ { - "expr": "max by(instance,device) ((irate(node_disk_write_time_seconds_total{ instance=~\"($ceph_hosts).*\"}[5m]) ) / clamp_min(irate(node_disk_writes_completed_total{ instance=~\"($ceph_hosts).*\"}[5m]), 0.001) or (irate(node_disk_read_time_seconds_total{ instance=~\"($ceph_hosts).*\"}[5m]) ) / clamp_min(irate(node_disk_reads_completed_total{ instance=~\"($ceph_hosts).*\"}[5m]), 0.001) ) * on(instance,device) group_left(ceph_daemon) label_replace(label_replace(ceph_disk_occupation{instance=~\"($ceph_hosts).*\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"(.*)\")", + "expr": "max by(instance,device) (label_replace((irate(node_disk_write_time_seconds_total{ instance=~\"($ceph_hosts).*\"}[5m]) ) / clamp_min(irate(node_disk_writes_completed_total{ instance=~\"($ceph_hosts).*\"}[5m]), 0.001) or (irate(node_disk_read_time_seconds_total{ instance=~\"($ceph_hosts).*\"}[5m]) ) / clamp_min(irate(node_disk_reads_completed_total{ instance=~\"($ceph_hosts).*\"}[5m]), 0.001), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")) * on(instance,device) group_left(ceph_daemon) label_replace(label_replace(ceph_disk_occupation{instance=~\"($ceph_hosts).*\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1083,7 +1083,7 @@ "steppedLine": false, "targets": [ { - "expr": "((irate(node_disk_io_time_ms{instance=~\"($ceph_hosts).*\"}[5m]) / 10 ) or irate(node_disk_io_time_seconds_total{instance=~\"($ceph_hosts).*\"}[5m]) * 100)* on(instance, device) group_left(ceph_daemon) label_replace(label_replace(ceph_disk_occupation{instance=~\"($ceph_hosts).*\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"(.*)\")", + "expr": "label_replace(((irate(node_disk_io_time_ms{instance=~\"($ceph_hosts).*\"}[5m]) / 10 ) or irate(node_disk_io_time_seconds_total{instance=~\"($ceph_hosts).*\"}[5m]) * 100), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") * on(instance, device) group_left(ceph_daemon) label_replace(label_replace(ceph_disk_occupation{instance=~\"($ceph_hosts).*\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1168,7 +1168,7 @@ "options": [], "query": "label_values(node_scrape_collector_success, instance) ", "refresh": 1, - "regex": "([^:]*).*", + "regex": "([^.:]*).*", "skipUrlSync": false, "sort": 3, "tagValuesQuery": "", diff --git a/ceph/monitoring/grafana/dashboards/hosts-overview.json b/ceph/monitoring/grafana/dashboards/hosts-overview.json index acaf9b740..d9f1fb29d 100644 --- a/ceph/monitoring/grafana/dashboards/hosts-overview.json +++ b/ceph/monitoring/grafana/dashboards/hosts-overview.json @@ -133,6 +133,7 @@ "datasource": "$datasource", "decimals": 0, "description": "Average CPU busy across all hosts (OSD, RGW, MON etc) within the cluster", + "decimals": 2, "format": "percentunit", "gauge": { "maxValue": 100, @@ -216,6 +217,7 @@ "datasource": "$datasource", "decimals": 0, "description": "Average Memory Usage across all hosts in the cluster (excludes buffer/cache usage)", + "decimals": 2, "format": "percentunit", "gauge": { "maxValue": 100, @@ -267,7 +269,7 @@ "tableColumn": "", "targets": [ { - "expr": "avg (((node_memory_MemTotal{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_MemTotal_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"})- (\n (node_memory_MemFree{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_MemFree_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}) + \n (node_memory_Cached{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_Cached_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}) + \n (node_memory_Buffers{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_Buffers_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}) +\n (node_memory_Slab{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_Slab_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"})\n )) /\n (node_memory_MemTotal{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_MemTotal_bytes{instance=~\"[[osd_hosts]]|[[rgw_hosts]]|[[mon_hosts]]|[[mds_hosts]].*\"} ))", + "expr": "avg (((node_memory_MemTotal{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_MemTotal_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"})- (\n (node_memory_MemFree{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_MemFree_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}) + \n (node_memory_Cached{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_Cached_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}) + \n (node_memory_Buffers{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_Buffers_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}) +\n (node_memory_Slab{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_Slab_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"})\n )) /\n (node_memory_MemTotal{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_MemTotal_bytes{instance=~\"($osd_hosts|$rgw_hosts|$mon_hosts|$mds_hosts).*\"} ))", "format": "time_series", "instant": true, "intervalFactor": 1, @@ -431,7 +433,7 @@ "tableColumn": "", "targets": [ { - "expr" : "avg (\n ((irate(node_disk_io_time_ms[5m]) / 10 ) or\n (irate(node_disk_io_time_seconds_total[5m]) * 100)\n ) *\n on(instance, device) label_replace(label_replace(ceph_disk_occupation{instance=~\"($osd_hosts).*\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"(.*)\")\n)", + "expr" : "avg (\n label_replace((irate(node_disk_io_time_ms[5m]) / 10 ) or\n (irate(node_disk_io_time_seconds_total[5m]) * 100), \"instance\", \"$1\", \"instance\", \"([^.:]*).*\"\n ) *\n on(instance, device) label_replace(label_replace(ceph_disk_occupation{instance=~\"($osd_hosts).*\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^.:]*).*\")\n)", "format": "time_series", "instant": true, "intervalFactor": 1, @@ -740,7 +742,7 @@ "multi": false, "name": "osd_hosts", "options": [], - "query": "label_values(ceph_disk_occupation, instance)", + "query": "label_values(ceph_disk_occupation, exported_instance)", "refresh": 1, "regex": "([^.]*).*", "skipUrlSync": false, diff --git a/ceph/monitoring/grafana/dashboards/osd-device-details.json b/ceph/monitoring/grafana/dashboards/osd-device-details.json index 6b70778df..8b819e902 100644 --- a/ceph/monitoring/grafana/dashboards/osd-device-details.json +++ b/ceph/monitoring/grafana/dashboards/osd-device-details.json @@ -390,14 +390,14 @@ "steppedLine": false, "targets": [ { - "expr": "(irate(node_disk_read_time_seconds_total[1m]) / irate(node_disk_reads_completed_total[1m]) and on (instance, device) label_replace(label_replace(ceph_disk_occupation{ceph_daemon=~\"$osd\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"(.*)\"))", + "expr": "(label_replace(irate(node_disk_read_time_seconds_total[1m]) / irate(node_disk_reads_completed_total[1m]), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") and on (instance, device) label_replace(label_replace(ceph_disk_occupation{ceph_daemon=~\"$osd\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\"))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}/{{device}} Reads", "refId": "A" }, { - "expr": "(irate(node_disk_write_time_seconds_total[1m]) / irate(node_disk_writes_completed_total[1m]) and on (instance, device) label_replace(label_replace(ceph_disk_occupation{ceph_daemon=~\"$osd\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"(.*)\"))", + "expr": "(label_replace(irate(node_disk_write_time_seconds_total[1m]) / irate(node_disk_writes_completed_total[1m]), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") and on (instance, device) label_replace(label_replace(ceph_disk_occupation{ceph_daemon=~\"$osd\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\"))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}/{{device}} Writes", @@ -486,14 +486,14 @@ "steppedLine": false, "targets": [ { - "expr": "irate(node_disk_writes_completed_total[1m]) and on (instance, device) label_replace(label_replace(ceph_disk_occupation{ceph_daemon=~\"$osd\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"(.*)\")", + "expr": "label_replace(irate(node_disk_writes_completed_total[1m]), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") and on (instance, device) label_replace(label_replace(ceph_disk_occupation{ceph_daemon=~\"$osd\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}} on {{instance}} Writes", "refId": "A" }, { - "expr": "irate(node_disk_reads_completed_total[1m]) and on (instance, device) label_replace(label_replace(ceph_disk_occupation{ceph_daemon=~\"$osd\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"(.*)\")", + "expr": "label_replace(irate(node_disk_reads_completed_total[1m]), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") and on (instance, device) label_replace(label_replace(ceph_disk_occupation{ceph_daemon=~\"$osd\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}} on {{instance}} Reads", @@ -582,14 +582,14 @@ "steppedLine": false, "targets": [ { - "expr": "irate(node_disk_read_bytes_total[1m]) and on (instance, device) label_replace(label_replace(ceph_disk_occupation{ceph_daemon=~\"$osd\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"(.*)\")", + "expr": "label_replace(irate(node_disk_read_bytes_total[1m]), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") and on (instance, device) label_replace(label_replace(ceph_disk_occupation{ceph_daemon=~\"$osd\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}} {{device}} Reads", "refId": "A" }, { - "expr": "irate(node_disk_written_bytes_total[1m]) and on (instance, device) label_replace(label_replace(ceph_disk_occupation{ceph_daemon=~\"$osd\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"(.*)\")", + "expr": "label_replace(irate(node_disk_written_bytes_total[1m]), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") and on (instance, device) label_replace(label_replace(ceph_disk_occupation{ceph_daemon=~\"$osd\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}} {{device}} Writes", @@ -673,7 +673,7 @@ "steppedLine": false, "targets": [ { - "expr": "irate(node_disk_io_time_seconds_total[1m]) and on (instance, device) label_replace(label_replace(ceph_disk_occupation{ceph_daemon=~\"$osd\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"(.*)\")", + "expr": "label_replace(irate(node_disk_io_time_seconds_total[1m]), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") and on (instance, device) label_replace(label_replace(ceph_disk_occupation{ceph_daemon=~\"$osd\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}} on {{instance}}", diff --git a/ceph/monitoring/grafana/dashboards/pool-detail.json b/ceph/monitoring/grafana/dashboards/pool-detail.json index 14e468a52..41554ed30 100644 --- a/ceph/monitoring/grafana/dashboards/pool-detail.json +++ b/ceph/monitoring/grafana/dashboards/pool-detail.json @@ -50,6 +50,7 @@ "#d44a3a" ], "datasource": "$datasource", + "decimals": 2, "format": "percentunit", "gauge": { "maxValue": 1, @@ -101,7 +102,7 @@ "tableColumn": "", "targets": [ { - "expr": "(ceph_pool_stored / ceph_pool_max_avail) * on(pool_id) group_left(instance,name) ceph_pool_metadata{name=~\"$pool_name\"}", + "expr": "(ceph_pool_stored / (ceph_pool_stored + ceph_pool_max_avail)) * on(pool_id) group_left(instance,name) ceph_pool_metadata{name=~\"$pool_name\"}", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -182,7 +183,7 @@ "tableColumn": "", "targets": [ { - "expr": "((ceph_pool_max_avail - ceph_pool_stored) / deriv(ceph_pool_stored[6h])) * on(pool_id) group_left(instance,name) ceph_pool_metadata{name=~\"$pool_name\"} > 0", + "expr": "(ceph_pool_max_avail / deriv(ceph_pool_stored[6h])) * on(pool_id) group_left(instance,name) ceph_pool_metadata{name=~\"$pool_name\"} > 0", "format": "time_series", "intervalFactor": 1, "refId": "A" diff --git a/ceph/monitoring/prometheus/alerts/ceph_default_alerts.yml b/ceph/monitoring/prometheus/alerts/ceph_default_alerts.yml index a9b0560d4..aebd81615 100644 --- a/ceph/monitoring/prometheus/alerts/ceph_default_alerts.yml +++ b/ceph/monitoring/prometheus/alerts/ceph_default_alerts.yml @@ -166,6 +166,7 @@ groups: rules: - alert: root volume full expr: node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"} * 100 < 5 + for: 5m labels: severity: critical type: ceph_default @@ -199,15 +200,10 @@ groups: Node {{ $labels.instance }} experiences packet errors > 1 packet/s on interface {{ $labels.device }}. - # predict fs fill-up times - - alert: storage filling + - alert: storage filling up expr: | - ( - ( - node_filesystem_free_bytes / deriv(node_filesystem_free_bytes[2d]) - * on(instance) group_left(nodename) node_uname_info - ) <= 5 - ) > 0 + predict_linear(node_filesystem_free_bytes[2d], 3600 * 24 * 5) * + on(instance) group_left(nodename) node_uname_info < 0 labels: severity: warning type: ceph_default @@ -234,10 +230,9 @@ groups: - alert: pool filling up expr: | ( - ( - (ceph_pool_max_avail - ceph_pool_stored) / deriv(ceph_pool_max_avail[2d]) - ) * on(pool_id) group_right ceph_pool_metadata <= 5 - ) > 0 + predict_linear(ceph_pool_stored[2d], 3600 * 24 * 5) >= + ceph_pool_max_avail + ) * on(pool_id) group_right(name) ceph_pool_metadata labels: severity: warning type: ceph_default diff --git a/ceph/qa/cephfs/begin.yaml b/ceph/qa/cephfs/begin.yaml index 0f3beb8fb..b69eac9c6 100644 --- a/ceph/qa/cephfs/begin.yaml +++ b/ceph/qa/cephfs/begin.yaml @@ -16,6 +16,11 @@ tasks: # for xfstests-dev - dump - indent + # for fsx + - libaio-dev + - libtool-bin + - uuid-dev + - xfslibs-dev rpm: - bison - flex @@ -30,4 +35,9 @@ tasks: - xfsdump - xfsprogs - xfsprogs-devel + # for fsx + - libaio-devel + - libtool + - libuuid-devel + - xfsprogs-devel - ceph: diff --git a/ceph/qa/distros/supported-random-distro$/centos_8.yaml b/ceph/qa/distros/supported-random-distro$/centos_8.yaml deleted file mode 120000 index b7e6c9b4e..000000000 --- a/ceph/qa/distros/supported-random-distro$/centos_8.yaml +++ /dev/null @@ -1 +0,0 @@ -../all/centos_8.yaml \ No newline at end of file diff --git a/ceph/qa/distros/supported-random-distro$/centos_latest.yaml b/ceph/qa/distros/supported-random-distro$/centos_latest.yaml new file mode 120000 index 000000000..591d55ba5 --- /dev/null +++ b/ceph/qa/distros/supported-random-distro$/centos_latest.yaml @@ -0,0 +1 @@ +../all/centos_8.1.yaml \ No newline at end of file diff --git a/ceph/qa/distros/supported-random-distro$/rhel_8.yaml b/ceph/qa/distros/supported-random-distro$/rhel_latest.yaml similarity index 100% rename from ceph/qa/distros/supported-random-distro$/rhel_8.yaml rename to ceph/qa/distros/supported-random-distro$/rhel_latest.yaml diff --git a/ceph/qa/distros/supported/centos_latest.yaml b/ceph/qa/distros/supported/centos_latest.yaml index b7e6c9b4e..591d55ba5 120000 --- a/ceph/qa/distros/supported/centos_latest.yaml +++ b/ceph/qa/distros/supported/centos_latest.yaml @@ -1 +1 @@ -../all/centos_8.yaml \ No newline at end of file +../all/centos_8.1.yaml \ No newline at end of file diff --git a/ceph/qa/overrides/short_pg_log.yaml b/ceph/qa/overrides/short_pg_log.yaml index 20cc101de..fa55e91ea 100644 --- a/ceph/qa/overrides/short_pg_log.yaml +++ b/ceph/qa/overrides/short_pg_log.yaml @@ -4,3 +4,4 @@ overrides: global: osd_min_pg_log_entries: 1 osd_max_pg_log_entries: 2 + osd_pg_log_trim_min: 0 diff --git a/ceph/qa/rgw_bucket_sharding/many.yaml b/ceph/qa/rgw_bucket_sharding/many.yaml deleted file mode 100644 index f42c9ad5f..000000000 --- a/ceph/qa/rgw_bucket_sharding/many.yaml +++ /dev/null @@ -1,5 +0,0 @@ -overrides: - ceph: - conf: - global: - rgw override bucket index max shards: 1999 diff --git a/ceph/qa/standalone/mon/mon-last-epoch-clean.sh b/ceph/qa/standalone/mon/mon-last-epoch-clean.sh new file mode 100755 index 000000000..e38663c6a --- /dev/null +++ b/ceph/qa/standalone/mon/mon-last-epoch-clean.sh @@ -0,0 +1,307 @@ +#!/usr/bin/env bash + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7302" # git grep '\<7105\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + + +function check_lec_equals_pools() { + + local pool_id=$1 + + report=$(ceph report) + lec=$(echo $report | \ + jq '.osdmap_clean_epochs.min_last_epoch_clean') + + if [[ -z "$pool_id" ]]; then + pools=($(echo $report | \ + jq \ + ".osdmap_clean_epochs.last_epoch_clean.per_pool[] |" \ + " select(.floor == $lec) | .poolid")) + + [[ ${#pools[*]} -eq 2 ]] || ( echo $report ; return 1 ) + else + floor=($(echo $report | \ + jq \ + ".osdmap_clean_epochs.last_epoch_clean.per_pool[] |" \ + " select(.poolid == $pool_id) | .floor")) + + [[ $lec -eq $floor ]] || ( echo $report ; return 1 ) + fi + return 0 +} + +function check_lec_lower_than_pool() { + + local pool_id=$1 + [[ -z "$pool_id" ]] && ( echo "expected pool_id as parameter" ; exit 1 ) + + report=$(ceph report) + lec=$(echo $report | \ + jq '.osdmap_clean_epochs.min_last_epoch_clean') + + floor=($(echo $report | \ + jq \ + ".osdmap_clean_epochs.last_epoch_clean.per_pool[] |" \ + " select(.poolid == $pool_id) | .floor")) + + [[ $lec -lt $floor ]] || ( echo $report ; return 1 ) + return 0 +} + +function check_floor_pool_greater_than_pool() { + + local pool_a=$1 + local pool_b=$1 + [[ -z "$pool_a" ]] && ( echo "expected id as first parameter" ; exit 1 ) + [[ -z "$pool_b" ]] && ( echo "expected id as second parameter" ; exit 1 ) + + report=$(ceph report) + + floor_a=($(echo $report | \ + jq \ + ".osdmap_clean_epochs.last_epoch_clean.per_pool[] |" \ + " select(.poolid == $pool_a) | .floor")) + + floor_b=($(echo $report | \ + jq \ + ".osdmap_clean_epochs.last_epoch_clean.per_pool[] |" \ + " select(.poolid == $pool_b) | .floor")) + + [[ $floor_a -gt $floor_b ]] || ( echo $report ; return 1 ) + return 0 +} + +function check_lec_honours_osd() { + + local osd=$1 + + report=$(ceph report) + lec=$(echo $report | \ + jq '.osdmap_clean_epochs.min_last_epoch_clean') + + if [[ -z "$osd" ]]; then + osds=($(echo $report | \ + jq \ + ".osdmap_clean_epochs.osd_epochs[] |" \ + " select(.epoch >= $lec) | .id")) + + [[ ${#osds[*]} -eq 3 ]] || ( echo $report ; return 1 ) + else + epoch=($(echo $report | \ + jq \ + ".osdmap_clean_epochs.osd_epochs[] |" \ + " select(.id == $id) | .epoch")) + [[ ${#epoch[*]} -eq 1 ]] || ( echo $report ; return 1 ) + [[ ${epoch[0]} -ge $lec ]] || ( echo $report ; return 1 ) + fi + + return 0 +} + +function validate_fc() { + report=$(ceph report) + lec=$(echo $report | \ + jq '.osdmap_clean_epochs.min_last_epoch_clean') + osdm_fc=$(echo $report | \ + jq '.osdmap_first_committed') + + [[ $lec -eq $osdm_fc ]] || ( echo $report ; return 1 ) + return 0 +} + +function get_fc_lc_diff() { + report=$(ceph report) + osdm_fc=$(echo $report | \ + jq '.osdmap_first_committed') + osdm_lc=$(echo $report | \ + jq '.osdmap_last_committed') + + echo $((osdm_lc - osdm_fc)) +} + +function get_pool_id() { + + local pn=$1 + [[ -z "$pn" ]] && ( echo "expected pool name as argument" ; exit 1 ) + + report=$(ceph report) + pool_id=$(echo $report | \ + jq ".osdmap.pools[] | select(.pool_name == \"$pn\") | .pool") + + [[ $pool_id -ge 0 ]] || \ + ( echo "unexpected pool id for pool \'$pn\': $pool_id" ; return -1 ) + + echo $pool_id + return 0 +} + +function wait_for_total_num_maps() { + # rip wait_for_health, becaue it's easier than deduplicating the code + local -a delays=($(get_timeout_delays $TIMEOUT .1)) + local -i loop=0 + local -i v_diff=$1 + + while [[ $(get_fc_lc_diff) -gt $v_diff ]]; do + if (( $loop >= ${#delays[*]} )) ; then + echo "maps were not trimmed" + return 1 + fi + sleep ${delays[$loop]} + loop+=1 + done +} + +function TEST_mon_last_clean_epoch() { + + local dir=$1 + + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + run_osd $dir 2 || return 1 + osd_pid=$(cat $dir/osd.2.pid) + + sleep 5 + + ceph tell osd.* injectargs '--osd-beacon-report-interval 10' || exit 1 + ceph tell mon.* injectargs \ + '--mon-min-osdmap-epochs 2 --paxos-service-trim-min 1' || exit 1 + + create_pool foo 32 + create_pool bar 32 + + foo_id=$(get_pool_id "foo") + bar_id=$(get_pool_id "bar") + + [[ $foo_id -lt 0 ]] && ( echo "couldn't find pool 'foo' id" ; exit 1 ) + [[ $bar_id -lt 0 ]] && ( echo "couldn't find pool 'bar' id" ; exit 1 ) + + # no real clue why we are getting these warnings, but let's make them go + # away so we can be happy. + + ceph osd set-full-ratio 0.97 + ceph osd set-backfillfull-ratio 0.97 + + wait_for_health_ok || exit 1 + + pre_map_diff=$(get_fc_lc_diff) + wait_for_total_num_maps 2 + post_map_diff=$(get_fc_lc_diff) + + [[ $post_map_diff -le $pre_map_diff ]] || exit 1 + + pre_map_diff=$post_map_diff + + ceph osd pool set foo size 3 + ceph osd pool set bar size 3 + + wait_for_health_ok || exit 1 + + check_lec_equals_pools || exit 1 + check_lec_honours_osd || exit 1 + validate_fc || exit 1 + + # down osd.2; expected result (because all pools' size equals 3): + # - number of committed maps increase over 2 + # - lec equals fc + # - lec equals osd.2's epoch + # - all pools have floor equal to lec + + while kill $osd_pid ; do sleep 1 ; done + ceph osd down 2 + sleep 5 # seriously, just to make sure things settle; we may not need this. + + # generate some maps + for ((i=0; i <= 10; ++i)); do + ceph osd set noup + sleep 1 + ceph osd unset noup + sleep 1 + done + + post_map_diff=$(get_fc_lc_diff) + [[ $post_map_diff -gt 2 ]] || exit 1 + + validate_fc || exit 1 + check_lec_equals_pools || exit 1 + check_lec_honours_osd 2 || exit 1 + + # adjust pool 'bar' size to 2; expect: + # - number of committed maps still over 2 + # - lec equals fc + # - lec equals pool 'foo' floor + # - pool 'bar' floor greater than pool 'foo' + + ceph osd pool set bar size 2 + + diff_ver=$(get_fc_lc_diff) + [[ $diff_ver -gt 2 ]] || exit 1 + + validate_fc || exit 1 + + check_lec_equals_pools $foo_id || exit 1 + check_lec_lower_than_pool $bar_id || exit 1 + + check_floor_pool_greater_than_pool $bar_id $foo_id || exit 1 + + # set pool 'foo' size to 2; expect: + # - health_ok + # - lec equals pools + # - number of committed maps decreases + # - lec equals fc + + pre_map_diff=$(get_fc_lc_diff) + + ceph osd pool set foo size 2 || exit 1 + wait_for_clean || exit 1 + + check_lec_equals_pools || exit 1 + validate_fc || exit 1 + + if ! wait_for_total_num_maps 2 ; then + post_map_diff=$(get_fc_lc_diff) + # number of maps is decreasing though, right? + [[ $post_map_diff -lt $pre_map_diff ]] || exit 1 + fi + + # bring back osd.2; expect: + # - health_ok + # - lec equals fc + # - number of committed maps equals 2 + # - all pools have floor equal to lec + + pre_map_diff=$(get_fc_lc_diff) + + activate_osd $dir 2 || exit 1 + wait_for_health_ok || exit 1 + validate_fc || exit 1 + check_lec_equals_pools || exit 1 + + if ! wait_for_total_num_maps 2 ; then + post_map_diff=$(get_fc_lc_diff) + # number of maps is decreasing though, right? + [[ $post_map_diff -lt $pre_map_diff ]] || exit 1 + fi + + return 0 +} + +main mon-last-clean-epoch "$@" diff --git a/ceph/qa/standalone/osd/osd-backfill-stats.sh b/ceph/qa/standalone/osd/osd-backfill-stats.sh index ea43b00fe..db8f4da66 100755 --- a/ceph/qa/standalone/osd/osd-backfill-stats.sh +++ b/ceph/qa/standalone/osd/osd-backfill-stats.sh @@ -353,6 +353,7 @@ function TEST_backfill_out2() { sleep 2 primary=$(get_primary $poolname obj1) ceph osd unset nobackfill + ceph tell osd.$primary get_latest_osdmap ceph tell osd.$primary debug kick_recovery_wq 0 sleep 2 @@ -410,6 +411,7 @@ function TEST_backfill_sizeup4_allout() { sleep 2 primary=$(get_primary $poolname obj1) ceph osd unset nobackfill + ceph tell osd.$primary get_latest_osdmap ceph tell osd.$primary debug kick_recovery_wq 0 sleep 2 @@ -474,6 +476,7 @@ function TEST_backfill_remapped() { primary=$(get_primary $poolname obj1) ceph osd unset nobackfill + ceph tell osd.$primary get_latest_osdmap ceph tell osd.$primary debug kick_recovery_wq 0 sleep 2 @@ -534,6 +537,7 @@ function TEST_backfill_ec_all_out() { sleep 2 primary=$(get_primary $poolname obj1) ceph osd unset nobackfill + ceph tell osd.$primary get_latest_osdmap ceph tell osd.$primary debug kick_recovery_wq 0 sleep 2 @@ -584,6 +588,7 @@ function TEST_backfill_ec_prim_out() { sleep 2 primary=$(get_primary $poolname obj1) ceph osd unset nobackfill + ceph tell osd.$primary get_latest_osdmap ceph tell osd.$primary debug kick_recovery_wq 0 sleep 2 @@ -642,6 +647,7 @@ function TEST_backfill_ec_down_all_out() { sleep 2 primary=$(get_primary $poolname obj1) ceph osd unset nobackfill + ceph tell osd.$primary get_latest_osdmap ceph tell osd.$primary debug kick_recovery_wq 0 sleep 2 flush_pg_stats @@ -726,6 +732,7 @@ function TEST_backfill_ec_down_out() { sleep 2 primary=$(get_primary $poolname obj1) ceph osd unset nobackfill + ceph tell osd.$primary get_latest_osdmap ceph tell osd.$primary debug kick_recovery_wq 0 sleep 2 diff --git a/ceph/qa/standalone/scrub/osd-scrub-repair.sh b/ceph/qa/standalone/scrub/osd-scrub-repair.sh index 12981dd8f..d5a5ee332 100755 --- a/ceph/qa/standalone/scrub/osd-scrub-repair.sh +++ b/ceph/qa/standalone/scrub/osd-scrub-repair.sh @@ -494,8 +494,9 @@ function TEST_auto_repair_bluestore_failed() { repair $pgid sleep 2 + flush_pg_stats ceph pg dump pgs - ceph pg dump pgs | grep -q "^$(pgid).* active+clean " || return 1 + ceph pg dump pgs | grep -q -e "^${pgid}.* active+clean " -e "^${pgid}.* active+clean+wait " || return 1 grep scrub_finish $dir/osd.${primary}.log # Tear down diff --git a/ceph/qa/suites/fs/basic_functional/tasks/cephfs_scrub_tests.yaml b/ceph/qa/suites/fs/basic_functional/tasks/cephfs_scrub_tests.yaml index 7fdee84b1..a86612149 100644 --- a/ceph/qa/suites/fs/basic_functional/tasks/cephfs_scrub_tests.yaml +++ b/ceph/qa/suites/fs/basic_functional/tasks/cephfs_scrub_tests.yaml @@ -1,6 +1,7 @@ overrides: ceph: log-whitelist: + - Replacing daemon mds - Scrub error on inode - Behind on trimming - Metadata damage detected diff --git a/ceph/qa/suites/fs/basic_workload/tasks/cfuse_workunit_suites_fsx.yaml b/ceph/qa/suites/fs/basic_workload/tasks/cfuse_workunit_suites_fsx.yaml index 4b45dbf92..b16cfb17d 100644 --- a/ceph/qa/suites/fs/basic_workload/tasks/cfuse_workunit_suites_fsx.yaml +++ b/ceph/qa/suites/fs/basic_workload/tasks/cfuse_workunit_suites_fsx.yaml @@ -1,16 +1,4 @@ tasks: -- install: - extra_system_packages: - deb: - - libaio-dev - - libtool-bin - - uuid-dev - - xfslibs-dev - rpm: - - libaio-devel - - libtool - - libuuid-devel - - xfsprogs-devel - check-counter: counters: mds: diff --git a/ceph/qa/suites/kcephfs/cephfs/tasks/kclient_workunit_suites_fsx.yaml b/ceph/qa/suites/kcephfs/cephfs/tasks/kclient_workunit_suites_fsx.yaml index 67e63ba2e..8b2b1ab5c 100644 --- a/ceph/qa/suites/kcephfs/cephfs/tasks/kclient_workunit_suites_fsx.yaml +++ b/ceph/qa/suites/kcephfs/cephfs/tasks/kclient_workunit_suites_fsx.yaml @@ -1,16 +1,4 @@ tasks: -- install: - extra_system_packages: - deb: - - libaio-dev - - libtool-bin - - uuid-dev - - xfslibs-dev - rpm: - - libaio-devel - - libtool - - libuuid-devel - - xfsprogs-devel - workunit: clients: all: diff --git a/ceph/qa/suites/multimds/basic/tasks/cfuse_workunit_suites_fsx.yaml b/ceph/qa/suites/multimds/basic/tasks/cfuse_workunit_suites_fsx.yaml index 67e63ba2e..8b2b1ab5c 100644 --- a/ceph/qa/suites/multimds/basic/tasks/cfuse_workunit_suites_fsx.yaml +++ b/ceph/qa/suites/multimds/basic/tasks/cfuse_workunit_suites_fsx.yaml @@ -1,16 +1,4 @@ tasks: -- install: - extra_system_packages: - deb: - - libaio-dev - - libtool-bin - - uuid-dev - - xfslibs-dev - rpm: - - libaio-devel - - libtool - - libuuid-devel - - xfsprogs-devel - workunit: clients: all: diff --git a/ceph/qa/suites/rados/cephadm/smoke-roleless/% b/ceph/qa/suites/rados/cephadm/smoke-roleless/% new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/rados/cephadm/smoke-roleless/.qa b/ceph/qa/suites/rados/cephadm/smoke-roleless/.qa new file mode 120000 index 000000000..fea2489fd --- /dev/null +++ b/ceph/qa/suites/rados/cephadm/smoke-roleless/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/smoke-roleless/distro b/ceph/qa/suites/rados/cephadm/smoke-roleless/distro new file mode 120000 index 000000000..d2dffb181 --- /dev/null +++ b/ceph/qa/suites/rados/cephadm/smoke-roleless/distro @@ -0,0 +1 @@ +../smoke/distro \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/smoke-roleless/start.yaml b/ceph/qa/suites/rados/cephadm/smoke-roleless/start.yaml new file mode 100644 index 000000000..2ca0d0e00 --- /dev/null +++ b/ceph/qa/suites/rados/cephadm/smoke-roleless/start.yaml @@ -0,0 +1,32 @@ +tasks: +- cephadm: + roleless: true +- cephadm.shell: + host.a: + - ceph orch status + - ceph orch ps + - ceph orch ls + - ceph orch host ls + - ceph orch device ls +roles: +- - host.a + - osd.0 + - osd.1 + - osd.2 + - osd.3 + - client.0 +- - host.b + - osd.4 + - osd.5 + - osd.6 + - osd.7 + - client.1 +openstack: +- volumes: # attached to each instance + count: 4 + size: 10 # GB +overrides: + ceph: + conf: + osd: + osd shutdown pgref assert: true diff --git a/ceph/qa/suites/rados/cephadm/upgrade/1-start.yaml b/ceph/qa/suites/rados/cephadm/upgrade/1-start.yaml index 5b5d29adf..56d68ff55 100644 --- a/ceph/qa/suites/rados/cephadm/upgrade/1-start.yaml +++ b/ceph/qa/suites/rados/cephadm/upgrade/1-start.yaml @@ -1,4 +1,4 @@ tasks: - cephadm: - image: quay.io/ceph-ci/ceph:wip-sage4-testing-2020-03-19-1456 - cephadm_branch: wip-sage4-testing-2020-03-19-1456 + image: docker.io/ceph/ceph:v15.2.0 + cephadm_branch: v15.2.0 diff --git a/ceph/qa/suites/rados/dashboard/tasks/dashboard.yaml b/ceph/qa/suites/rados/dashboard/tasks/dashboard.yaml index bf9964481..e61294eee 100644 --- a/ceph/qa/suites/rados/dashboard/tasks/dashboard.yaml +++ b/ceph/qa/suites/rados/dashboard/tasks/dashboard.yaml @@ -33,6 +33,7 @@ tasks: - tasks.mgr.dashboard.test_auth - tasks.mgr.dashboard.test_cephfs - tasks.mgr.dashboard.test_cluster_configuration + - tasks.mgr.dashboard.test_crush_rule - tasks.mgr.dashboard.test_erasure_code_profile - tasks.mgr.dashboard.test_ganesha - tasks.mgr.dashboard.test_health diff --git a/ceph/qa/suites/rados/mgr/tasks/module_selftest.yaml b/ceph/qa/suites/rados/mgr/tasks/module_selftest.yaml index 17fa6b395..9fa956b7e 100644 --- a/ceph/qa/suites/rados/mgr/tasks/module_selftest.yaml +++ b/ceph/qa/suites/rados/mgr/tasks/module_selftest.yaml @@ -18,6 +18,7 @@ tasks: - influxdb python module not found - \(MGR_ZABBIX_ - foo bar + - Failed to open Telegraf - cephfs_test_runner: modules: - tasks.mgr.test_module_selftest diff --git a/ceph/qa/suites/rados/rest/centos_latest.yaml b/ceph/qa/suites/rados/rest/centos_latest.yaml deleted file mode 120000 index bd9854e70..000000000 --- a/ceph/qa/suites/rados/rest/centos_latest.yaml +++ /dev/null @@ -1 +0,0 @@ -.qa/distros/supported/centos_latest.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rados/rest/supported-random-distro$ b/ceph/qa/suites/rados/rest/supported-random-distro$ new file mode 120000 index 000000000..7cef21eef --- /dev/null +++ b/ceph/qa/suites/rados/rest/supported-random-distro$ @@ -0,0 +1 @@ +../basic/supported-random-distro$ \ No newline at end of file diff --git a/ceph/qa/suites/rados/singleton/all/osd-recovery.yaml b/ceph/qa/suites/rados/singleton/all/osd-recovery.yaml index e63536493..a5ffc3067 100644 --- a/ceph/qa/suites/rados/singleton/all/osd-recovery.yaml +++ b/ceph/qa/suites/rados/singleton/all/osd-recovery.yaml @@ -27,5 +27,6 @@ tasks: conf: osd: osd min pg log entries: 5 + osd pg log trim min: 0 osd_fast_fail_on_connection_refused: false - osd_recovery: diff --git a/ceph/qa/suites/rados/thrash/workloads/radosbench-high-concurrency.yaml b/ceph/qa/suites/rados/thrash/workloads/radosbench-high-concurrency.yaml new file mode 100644 index 000000000..902c4b56a --- /dev/null +++ b/ceph/qa/suites/rados/thrash/workloads/radosbench-high-concurrency.yaml @@ -0,0 +1,49 @@ +overrides: + ceph: + conf: + client.0: + debug ms: 1 + debug objecter: 20 + debug rados: 20 +tasks: +- full_sequential: + - radosbench: + clients: [client.0] + concurrency: 128 + size: 8192 + time: 90 + - radosbench: + clients: [client.0] + concurrency: 128 + size: 8192 + time: 90 + - radosbench: + clients: [client.0] + concurrency: 128 + size: 8192 + time: 90 + - radosbench: + clients: [client.0] + concurrency: 128 + size: 8192 + time: 90 + - radosbench: + clients: [client.0] + concurrency: 128 + size: 8192 + time: 90 + - radosbench: + clients: [client.0] + concurrency: 128 + size: 8192 + time: 90 + - radosbench: + clients: [client.0] + concurrency: 128 + size: 8192 + time: 90 + - radosbench: + clients: [client.0] + concurrency: 128 + size: 8192 + time: 90 diff --git a/ceph/qa/suites/rgw/crypt/3-rgw/rgw.yaml b/ceph/qa/suites/rgw/crypt/3-rgw/rgw.yaml index 4d14dcf04..ee8d62af0 100644 --- a/ceph/qa/suites/rgw/crypt/3-rgw/rgw.yaml +++ b/ceph/qa/suites/rgw/crypt/3-rgw/rgw.yaml @@ -2,6 +2,8 @@ overrides: ceph: conf: client: + setuser: ceph + setgroup: ceph rgw crypt require ssl: false debug rgw: 20 diff --git a/ceph/qa/suites/rgw/hadoop-s3a/overrides.yaml b/ceph/qa/suites/rgw/hadoop-s3a/overrides.yaml new file mode 100644 index 000000000..d52080bb5 --- /dev/null +++ b/ceph/qa/suites/rgw/hadoop-s3a/overrides.yaml @@ -0,0 +1,6 @@ +overrides: + ceph: + conf: + client: + setuser: ceph + setgroup: ceph diff --git a/ceph/qa/suites/rgw/multifs/overrides.yaml b/ceph/qa/suites/rgw/multifs/overrides.yaml index 038a9c9c6..1cb489072 100644 --- a/ceph/qa/suites/rgw/multifs/overrides.yaml +++ b/ceph/qa/suites/rgw/multifs/overrides.yaml @@ -3,6 +3,8 @@ overrides: wait-for-scrub: false conf: client: + setuser: ceph + setgroup: ceph debug rgw: 20 rgw crypt s3 kms backend: testing rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= diff --git a/ceph/qa/suites/rgw/multisite/overrides.yaml b/ceph/qa/suites/rgw/multisite/overrides.yaml index 9c383fc78..54e3db24f 100644 --- a/ceph/qa/suites/rgw/multisite/overrides.yaml +++ b/ceph/qa/suites/rgw/multisite/overrides.yaml @@ -3,6 +3,8 @@ overrides: wait-for-scrub: false conf: client: + setuser: ceph + setgroup: ceph debug rgw: 20 rgw crypt s3 kms backend: testing rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= diff --git a/ceph/qa/suites/rgw/singleton/overrides.yaml b/ceph/qa/suites/rgw/singleton/overrides.yaml index ed4ad591d..d2aea790a 100644 --- a/ceph/qa/suites/rgw/singleton/overrides.yaml +++ b/ceph/qa/suites/rgw/singleton/overrides.yaml @@ -3,4 +3,6 @@ overrides: wait-for-scrub: false conf: client: + setuser: ceph + setgroup: ceph debug rgw: 20 diff --git a/ceph/qa/suites/rgw/tempest/overrides.yaml b/ceph/qa/suites/rgw/tempest/overrides.yaml new file mode 100644 index 000000000..d52080bb5 --- /dev/null +++ b/ceph/qa/suites/rgw/tempest/overrides.yaml @@ -0,0 +1,6 @@ +overrides: + ceph: + conf: + client: + setuser: ceph + setgroup: ceph diff --git a/ceph/qa/suites/rgw/thrash/civetweb.yaml b/ceph/qa/suites/rgw/thrash/civetweb.yaml index 5845a0e6c..db4b04339 100644 --- a/ceph/qa/suites/rgw/thrash/civetweb.yaml +++ b/ceph/qa/suites/rgw/thrash/civetweb.yaml @@ -1,3 +1,8 @@ overrides: + ceph: + conf: + client: + setuser: ceph + setgroup: ceph rgw: frontend: civetweb diff --git a/ceph/qa/suites/rgw/verify/overrides.yaml b/ceph/qa/suites/rgw/verify/overrides.yaml index 12d32beca..aefee7b70 100644 --- a/ceph/qa/suites/rgw/verify/overrides.yaml +++ b/ceph/qa/suites/rgw/verify/overrides.yaml @@ -2,6 +2,8 @@ overrides: ceph: conf: client: + setuser: ceph + setgroup: ceph debug rgw: 20 rgw crypt s3 kms backend: testing rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= diff --git a/ceph/qa/suites/rgw/website/overrides.yaml b/ceph/qa/suites/rgw/website/overrides.yaml index 57fb5d033..0eb474980 100644 --- a/ceph/qa/suites/rgw/website/overrides.yaml +++ b/ceph/qa/suites/rgw/website/overrides.yaml @@ -11,6 +11,8 @@ overrides: osd_min_pg_log_entries: 10 osd_max_pg_log_entries: 10 client: + setuser: ceph + setgroup: ceph debug rgw: 20 rgw crypt s3 kms backend: testing rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/.qa b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/% b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/% new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/.qa b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/+ b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/+ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/.qa b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/openstack.yaml b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/openstack.yaml new file mode 100644 index 000000000..b0f3b9b4d --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/openstack.yaml @@ -0,0 +1,4 @@ +openstack: + - volumes: # attached to each instance + count: 4 + size: 30 # GB diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/start.yaml b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/start.yaml new file mode 100644 index 000000000..c631b0ed2 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/0-cluster/start.yaml @@ -0,0 +1,21 @@ +meta: +- desc: | + Insatll and run ceph on one node, + with a separate client 1. + Upgrade client 1 to octopus + Run tests against old cluster +roles: +- - mon.a + - mon.b + - mon.c + - osd.0 + - osd.1 + - osd.2 + - client.0 + - mgr.x +- - client.1 +overrides: + ceph: + #log-whitelist: + #- failed to encode map + fs: xfs diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/.qa b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/nautilus-client-x.yaml b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/nautilus-client-x.yaml new file mode 100644 index 000000000..e8758027f --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/1-install/nautilus-client-x.yaml @@ -0,0 +1,11 @@ +tasks: +- install: + branch: octopus + exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev','python34-cephfs','python34-rados'] +- print: "**** done install octopus" +- install.upgrade: + exclude_packages: ['ceph-test', 'ceph-test-dbg','libcephfs1', 'python-ceph'] + client.1: +- print: "**** done install.upgrade to -x on client.0" +- ceph: +- print: "**** done ceph task" diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/.qa b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/defaults.yaml b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/defaults.yaml new file mode 100644 index 000000000..dff6623ad --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/defaults.yaml @@ -0,0 +1,6 @@ +overrides: + ceph: + conf: + client: + rbd default features: 61 + diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/layering.yaml b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/layering.yaml new file mode 100644 index 000000000..5613d0155 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/2-features/layering.yaml @@ -0,0 +1,6 @@ +overrides: + ceph: + conf: + client: + rbd default features: 1 + diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/.qa b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/rbd_notification_tests.yaml b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/rbd_notification_tests.yaml new file mode 100644 index 000000000..10b86530a --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/3-workload/rbd_notification_tests.yaml @@ -0,0 +1,36 @@ +tasks: +- parallel: + - workunit: + branch: octopus + clients: + client.0: + - rbd/notify_master.sh + env: + RBD_FEATURES: "61" + - workunit: + #The line below to change to 'pacific' + branch: master + clients: + client.1: + - rbd/notify_slave.sh + env: + RBD_FEATURES: "61" +- print: "**** done rbd: old librbd -> new librbd" +- parallel: + - workunit: + #The line below to change to 'pacific' + branch: master + clients: + client.0: + - rbd/notify_slave.sh + env: + RBD_FEATURES: "61" + - workunit: + #The line below to change to 'pacific' + branch: master + clients: + client.1: + - rbd/notify_master.sh + env: + RBD_FEATURES: "61" +- print: "**** done rbd: new librbd -> old librbd" diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/.qa b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/ubuntu_18.04.yaml b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/ubuntu_18.04.yaml new file mode 120000 index 000000000..886e87fa2 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-octopus-pacific/octopus-client-x/rbd/supported/ubuntu_18.04.yaml @@ -0,0 +1 @@ +../../../../../../distros/all/ubuntu_18.04.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-parallel/% b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-parallel/% new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-parallel/point-to-point-upgrade.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-parallel/point-to-point-upgrade.yaml new file mode 100644 index 000000000..1bcca4e75 --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-parallel/point-to-point-upgrade.yaml @@ -0,0 +1,177 @@ +meta: +- desc: | + Run ceph on two nodes, using one of them as a client, + with a separate client-only node. + Use xfs beneath the osds. + install ceph/octopus v15.2.1 point version + run workload and upgrade-sequence in parallel + (every point release should be tested) + run workload and upgrade-sequence in parallel + install ceph/octopus latest version + run workload and upgrade-sequence in parallel +overrides: + ceph: + log-whitelist: + - reached quota + - scrub + - osd_map_max_advance + - wrongly marked + - FS_DEGRADED + - POOL_APP_NOT_ENABLED + - CACHE_POOL_NO_HIT_SET + - POOL_FULL + - SMALLER_PG + - pool\(s\) full + - OSD_DOWN + - missing hit_sets + - CACHE_POOL_NEAR_FULL + - PG_AVAILABILITY + - PG_DEGRADED + - application not enabled + - cache pools at or near target size + - filesystem is degraded + - OBJECT_MISPLACED + ### ref: https://tracker.ceph.com/issues/40251 + #removed see ^ - failed to encode map + + fs: xfs + + conf: + global: + mon_warn_on_pool_no_app: false + mon: + mon debug unsafe allow tier with nonempty snaps: true + osd: + osd map max advance: 1000 + osd_class_default_list: "*" + osd_class_load_list: "*" + client: + rgw_crypt_require_ssl: false + rgw crypt s3 kms backend: testing + rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= +roles: +- - mon.a + - mds.a + - osd.0 + - osd.1 + - osd.2 + - mgr.x +- - mon.b + - mon.c + - osd.3 + - osd.4 + - osd.5 + - client.0 +- - client.1 +openstack: +- volumes: # attached to each instance + count: 3 + size: 30 # GB +tasks: +- print: "**** done octopus v15.2.1 about to install" +- install: + tag: v15.2.1 + # line below can be removed its from jewel test + #exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev', 'librgw2'] +- print: "**** done v15.2.1 install" +- ceph: + fs: xfs + add_osds_to_crush: true +- print: "**** done ceph xfs" +- sequential: + - workload +- print: "**** done workload v15.2.1" + + +####### upgrade to v15.2.?? PLACEHOLDER +#- install.upgrade: +# #exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev'] +# mon.a: +# tag: v15.2.?? +# mon.b: +# tag: v15.2.?? +# # Note that client.a IS NOT upgraded at this point +#- parallel: +# - workload_octopus +# - upgrade-sequence_octopus +#- print: "**** done parallel octopus v15.2.??" + +#### upgrade to latest octopus +- install.upgrade: + mon.a: + mon.b: +- parallel: + - workload_octopus + - upgrade-sequence_octopus +- print: "**** done parallel octopus branch" + +####################### +workload: + sequential: + - workunit: + clients: + client.0: + - suites/blogbench.sh +workload_octopus: + full_sequential: + - workunit: + branch: octopus + #tag: v15.2.1 + clients: + client.1: + - rados/test.sh + - cls + env: + CLS_RBD_GTEST_FILTER: '*:-TestClsRbd.snapshots_namespaces' + - print: "**** done rados/test.sh & cls workload_octopus" + - sequential: + - rgw: [client.0] + - print: "**** done rgw workload_octopus" + - s3tests: + client.0: + force-branch: ceph-octopus + rgw_server: client.0 + scan_for_encryption_keys: false + - print: "**** done s3tests workload_octopus" + - rbd_fsx: + clients: [client.0] + size: 134217728 + - print: "**** done rbd_fsx workload_octopus" + +upgrade-sequence_octopus: + sequential: + - print: "**** done branch: octopus install.upgrade" + - ceph.restart: [mds.a] + - sleep: + duration: 60 + - ceph.restart: [osd.0] + - sleep: + duration: 30 + - ceph.restart: [osd.1] + - sleep: + duration: 30 + - ceph.restart: [osd.2] + - sleep: + duration: 30 + - ceph.restart: [osd.3] + - sleep: + duration: 30 + - ceph.restart: [osd.4] + - sleep: + duration: 30 + - ceph.restart: [osd.5] + - sleep: + duration: 60 + - ceph.restart: [mgr.x] + - sleep: + duration: 60 + - ceph.restart: [mon.a] + - sleep: + duration: 60 + - ceph.restart: [mon.b] + - sleep: + duration: 60 + - ceph.restart: [mon.c] + - sleep: + duration: 60 + - print: "**** done ceph.restart all octopus branch mds/osd/mon" diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-parallel/supported-all-distro/ubuntu_latest.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-parallel/supported-all-distro/ubuntu_latest.yaml new file mode 100644 index 000000000..4d4464884 --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-parallel/supported-all-distro/ubuntu_latest.yaml @@ -0,0 +1,2 @@ +os_type: ubuntu +os_version: "18.04" diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/% b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/% new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/0-cluster/+ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/0-cluster/+ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/0-cluster/openstack.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/0-cluster/openstack.yaml new file mode 100644 index 000000000..5caffc353 --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/0-cluster/openstack.yaml @@ -0,0 +1,6 @@ +openstack: + - machine: + disk: 100 # GB + - volumes: # attached to each instance + count: 4 + size: 30 # GB diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/0-cluster/start.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/0-cluster/start.yaml new file mode 100644 index 000000000..5ebce3ced --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/0-cluster/start.yaml @@ -0,0 +1,33 @@ +meta: +- desc: | + Run ceph on two nodes, + with a separate client-only node. + Use xfs beneath the osds. +overrides: + ceph: + fs: xfs + log-whitelist: + - overall HEALTH_ + - \(MON_DOWN\) + - \(MGR_DOWN\) + ### ref: https://tracker.ceph.com/issues/40251 + #removed see ^ - failed to encode map + conf: + global: + enable experimental unrecoverable data corrupting features: "*" + mon: + mon warn on osd down out interval zero: false +roles: +- - mon.a + - mon.b + - mon.c + - mgr.x + - osd.0 + - osd.1 + - osd.2 + - osd.3 +- - osd.4 + - osd.5 + - osd.6 + - osd.7 +- - client.0 diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/1-ceph-install/octopus.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/1-ceph-install/octopus.yaml new file mode 100644 index 000000000..e42fd352c --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/1-ceph-install/octopus.yaml @@ -0,0 +1,19 @@ +meta: +- desc: install ceph/octopus v15.2.1 +tasks: +- install: + tag: v15.2.1 + exclude_packages: ['librados3'] + extra_packages: ['librados2'] +- print: "**** done install octopus v15.2.1" +- ceph: +- exec: + osd.0: + - ceph osd require-osd-release octopus + - ceph osd set-require-min-compat-client octopus +- print: "**** done ceph" +overrides: + ceph: + conf: + mon: + mon warn on osd down out interval zero: false diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/1.1.short_pg_log.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/1.1.short_pg_log.yaml new file mode 100644 index 000000000..20cc101de --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/1.1.short_pg_log.yaml @@ -0,0 +1,6 @@ +overrides: + ceph: + conf: + global: + osd_min_pg_log_entries: 1 + osd_max_pg_log_entries: 2 diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/2-partial-upgrade/firsthalf.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/2-partial-upgrade/firsthalf.yaml new file mode 100644 index 000000000..02ba5c1bb --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/2-partial-upgrade/firsthalf.yaml @@ -0,0 +1,13 @@ +meta: +- desc: | + install upgrade ceph/-x on one node only + 1st half + restart : osd.0,1,2,3 +tasks: +- install.upgrade: + osd.0: +- print: "**** done install.upgrade osd.0" +- ceph.restart: + daemons: [mon.a,mon.b,mon.c,mgr.x,osd.0,osd.1,osd.2,osd.3] + mon-health-to-clog: false +- print: "**** done ceph.restart 1st half" diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/3-thrash/default.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/3-thrash/default.yaml new file mode 100644 index 000000000..49e6f84f8 --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/3-thrash/default.yaml @@ -0,0 +1,27 @@ +meta: +- desc: | + randomly kill and revive osd + small chance to increase the number of pgs +overrides: + ceph: + log-whitelist: + - but it is still running + - wrongly marked me down + - objects unfound and apparently lost + - log bound mismatch + ### ref: https://tracker.ceph.com/issues/40251 + - failed to encode map +tasks: +- parallel: + - stress-tasks +stress-tasks: +- thrashosds: + timeout: 1200 + chance_pgnum_grow: 1 + chance_pgpnum_fix: 1 + chance_thrash_cluster_full: 0 + chance_thrash_pg_upmap: 0 + chance_thrash_pg_upmap_items: 0 + disable_objectstore_tool_tests: true + chance_force_recovery: 0 +- print: "**** done thrashosds 3-thrash" diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/+ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/+ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/fsx.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/fsx.yaml new file mode 100644 index 000000000..fd4081f23 --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/fsx.yaml @@ -0,0 +1,8 @@ +meta: +- desc: | + run basic fsx tests for rbd +stress-tasks: +- rbd_fsx: + clients: [client.0] + size: 134217728 +- print: "**** done rbd_fsx 4-workload" diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/radosbench.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/radosbench.yaml new file mode 100644 index 000000000..c545936c0 --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/radosbench.yaml @@ -0,0 +1,52 @@ +meta: +- desc: | + run randomized correctness test for rados operations + generate write load with rados bench +stress-tasks: +- full_sequential: + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 + - radosbench: + clients: [client.0] + time: 90 +- print: "**** done radosbench 4-workload" diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/rbd-cls.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/rbd-cls.yaml new file mode 100644 index 000000000..86c04338c --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/rbd-cls.yaml @@ -0,0 +1,10 @@ +meta: +- desc: | + run basic cls tests for rbd +stress-tasks: +- workunit: + branch: octopus + clients: + client.0: + - cls/test_cls_rbd.sh +- print: "**** done cls/test_cls_rbd.sh 4-workload" diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/rbd-import-export.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/rbd-import-export.yaml new file mode 100644 index 000000000..2f665b8d7 --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/rbd-import-export.yaml @@ -0,0 +1,12 @@ +meta: +- desc: | + run basic import/export cli tests for rbd +stress-tasks: +- workunit: + branch: octopus + clients: + client.0: + - rbd/import_export.sh + env: + RBD_CREATE_ARGS: --new-format +- print: "**** done rbd/import_export.sh 4-workload" diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/rbd_api.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/rbd_api.yaml new file mode 100644 index 000000000..95c820161 --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/rbd_api.yaml @@ -0,0 +1,10 @@ +meta: +- desc: | + librbd C and C++ api tests +stress-tasks: +- workunit: + branch: octopus + clients: + client.0: + - rbd/test_librbd.sh +- print: "**** done rbd/test_librbd.sh 4-workload" diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/readwrite.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/readwrite.yaml new file mode 100644 index 000000000..456868998 --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/readwrite.yaml @@ -0,0 +1,16 @@ +meta: +- desc: | + randomized correctness test for rados operations on a replicated pool, + using only reads, writes, and deletes +stress-tasks: +- full_sequential: + - rados: + clients: [client.0] + ops: 4000 + objects: 500 + write_append_excl: false + op_weights: + read: 45 + write: 45 + delete: 10 +- print: "**** done rados/readwrite 4-workload" diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/snaps-few-objects.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/snaps-few-objects.yaml new file mode 100644 index 000000000..ae232d867 --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/4-workload/snaps-few-objects.yaml @@ -0,0 +1,18 @@ +meta: +- desc: | + randomized correctness test for rados operations on a replicated pool with snapshot operations +stress-tasks: +- full_sequential: + - rados: + clients: [client.0] + ops: 4000 + objects: 50 + write_append_excl: false + op_weights: + read: 100 + write: 100 + delete: 50 + snap_create: 50 + snap_remove: 50 + rollback: 50 +- print: "**** done rados/snaps-few-objects 4-workload" diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/5-finish-upgrade.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/5-finish-upgrade.yaml new file mode 100644 index 000000000..803737c72 --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/5-finish-upgrade.yaml @@ -0,0 +1,8 @@ +tasks: +- install.upgrade: + osd.4: + client.0: +- ceph.restart: + daemons: [osd.4, osd.5, osd.6, osd.7] + wait-for-healthy: false + wait-for-osds-up: true diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/6-final-workload/+ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/6-final-workload/+ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/6-final-workload/rbd-python.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/6-final-workload/rbd-python.yaml new file mode 100644 index 000000000..b03f5fab7 --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/6-final-workload/rbd-python.yaml @@ -0,0 +1,10 @@ +meta: +- desc: | + librbd python api tests +tasks: +- workunit: + tag: v15.2.1 + clients: + client.0: + - rbd/test_librbd_python.sh +- print: "**** done rbd/test_librbd_python.sh 7-workload" diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/6-final-workload/snaps-many-objects.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/6-final-workload/snaps-many-objects.yaml new file mode 100644 index 000000000..805bf97c3 --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/6-final-workload/snaps-many-objects.yaml @@ -0,0 +1,16 @@ +meta: +- desc: | + randomized correctness test for rados operations on a replicated pool with snapshot operations +tasks: +- rados: + clients: [client.0] + ops: 4000 + objects: 500 + write_append_excl: false + op_weights: + read: 100 + write: 100 + delete: 50 + snap_create: 50 + snap_remove: 50 + rollback: 50 diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/objectstore/bluestore-bitmap.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/objectstore/bluestore-bitmap.yaml new file mode 100644 index 000000000..b18e04bee --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/objectstore/bluestore-bitmap.yaml @@ -0,0 +1,43 @@ +overrides: + thrashosds: + bdev_inject_crash: 2 + bdev_inject_crash_probability: .5 + ceph: + fs: xfs + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore fsck on mount: true + bluestore allocator: bitmap + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 +# this doesn't work with failures bc the log writes are not atomic across the two backends +# bluestore bluefs env mirror: true + bdev enable discard: true + bdev async discard: true + ceph-deploy: + fs: xfs + bluestore: yes + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore fsck on mount: true + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 + bdev enable discard: true + bdev async discard: true + diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/objectstore/bluestore-comp.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/objectstore/bluestore-comp.yaml new file mode 100644 index 000000000..b408032fd --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/objectstore/bluestore-comp.yaml @@ -0,0 +1,23 @@ +overrides: + thrashosds: + bdev_inject_crash: 2 + bdev_inject_crash_probability: .5 + ceph: + fs: xfs + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore compression mode: aggressive + bluestore fsck on mount: true + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 + +# this doesn't work with failures bc the log writes are not atomic across the two backends +# bluestore bluefs env mirror: true diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/objectstore/bluestore-stupid.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/objectstore/bluestore-stupid.yaml new file mode 100644 index 000000000..ca811f131 --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/objectstore/bluestore-stupid.yaml @@ -0,0 +1,43 @@ +overrides: + thrashosds: + bdev_inject_crash: 2 + bdev_inject_crash_probability: .5 + ceph: + fs: xfs + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore fsck on mount: true + bluestore allocator: stupid + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 +# this doesn't work with failures bc the log writes are not atomic across the two backends +# bluestore bluefs env mirror: true + bdev enable discard: true + bdev async discard: true + ceph-deploy: + fs: xfs + bluestore: yes + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore fsck on mount: true + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 + bdev enable discard: true + bdev async discard: true + diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/objectstore/filestore-xfs.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/objectstore/filestore-xfs.yaml new file mode 100644 index 000000000..f7aa0dd79 --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/objectstore/filestore-xfs.yaml @@ -0,0 +1,15 @@ +overrides: + ceph: + fs: xfs + conf: + osd: + osd objectstore: filestore + osd sloppy crc: true + ceph-deploy: + fs: xfs + filestore: True + conf: + osd: + osd objectstore: filestore + osd sloppy crc: true + diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/supported-all-distro/ubuntu_latest.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/supported-all-distro/ubuntu_latest.yaml new file mode 100644 index 000000000..4d4464884 --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/supported-all-distro/ubuntu_latest.yaml @@ -0,0 +1,2 @@ +os_type: ubuntu +os_version: "18.04" diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/thrashosds-health.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/thrashosds-health.yaml new file mode 100644 index 000000000..914f6e25e --- /dev/null +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-stress-split/thrashosds-health.yaml @@ -0,0 +1,15 @@ +overrides: + ceph: + log-whitelist: + - overall HEALTH_ + - \(OSDMAP_FLAGS\) + - \(OSD_ + - \(PG_ + - \(POOL_ + - \(CACHE_POOL_ + - \(SMALLER_PGP_NUM\) + - \(OBJECT_ + - \(SLOW_OPS\) + - \(REQUEST_SLOW\) + - \(TOO_FEW_PGS\) + - slow request diff --git a/ceph/qa/tasks/cephadm.py b/ceph/qa/tasks/cephadm.py index 2b076053a..2c64ca997 100644 --- a/ceph/qa/tasks/cephadm.py +++ b/ceph/qa/tasks/cephadm.py @@ -291,25 +291,11 @@ def ceph_bootstrap(ctx, config): testdir = teuthology.get_testdir(ctx) fsid = ctx.ceph[cluster_name].fsid + bootstrap_remote = ctx.ceph[cluster_name].bootstrap_remote + first_mon = ctx.ceph[cluster_name].first_mon + first_mon_role = ctx.ceph[cluster_name].first_mon_role mons = ctx.ceph[cluster_name].mons - first_mon_role = sorted(mons.keys())[0] - _, _, first_mon = teuthology.split_role(first_mon_role) - (bootstrap_remote,) = ctx.cluster.only(first_mon_role).remotes.keys() - log.info('First mon is mon.%s on %s' % (first_mon, - bootstrap_remote.shortname)) - ctx.ceph[cluster_name].bootstrap_remote = bootstrap_remote - ctx.ceph[cluster_name].first_mon = first_mon - - others = ctx.cluster.remotes[bootstrap_remote] - log.info('others %s' % others) - mgrs = sorted([r for r in others - if teuthology.is_type('mgr', cluster_name)(r)]) - if not mgrs: - raise RuntimeError('no mgrs on the same host as first mon %s' % first_mon) - _, _, first_mgr = teuthology.split_role(mgrs[0]) - log.info('First mgr is %s' % (first_mgr)) - ctx.ceph[cluster_name].first_mgr = first_mgr - + ctx.cluster.run(args=[ 'sudo', 'mkdir', '-p', '/etc/ceph', ]); @@ -338,14 +324,16 @@ def ceph_bootstrap(ctx, config): wait=False, started=True, ) - ctx.daemons.register_daemon( - bootstrap_remote, 'mgr', first_mgr, - cluster=cluster_name, - fsid=fsid, - logger=log.getChild('mgr.' + first_mgr), - wait=False, - started=True, - ) + if not ctx.ceph[cluster_name].roleless: + first_mgr = ctx.ceph[cluster_name].first_mgr + ctx.daemons.register_daemon( + bootstrap_remote, 'mgr', first_mgr, + cluster=cluster_name, + fsid=fsid, + logger=log.getChild('mgr.' + first_mgr), + wait=False, + started=True, + ) # bootstrap log.info('Bootstrapping...') @@ -356,16 +344,19 @@ def ceph_bootstrap(ctx, config): '-v', 'bootstrap', '--fsid', fsid, - '--mon-id', first_mon, - '--mgr-id', first_mgr, - '--orphan-initial-daemons', # we will do it explicitly! - '--skip-monitoring-stack', # we'll provision these explicitly '--config', '{}/seed.{}.conf'.format(testdir, cluster_name), '--output-config', '/etc/ceph/{}.conf'.format(cluster_name), '--output-keyring', '/etc/ceph/{}.client.admin.keyring'.format(cluster_name), '--output-pub-ssh-key', '{}/{}.pub'.format(testdir, cluster_name), ] + if not ctx.ceph[cluster_name].roleless: + cmd += [ + '--mon-id', first_mon, + '--mgr-id', first_mgr, + '--orphan-initial-daemons', # we will do it explicitly! + '--skip-monitoring-stack', # we'll provision these explicitly + ] if mons[first_mon_role].startswith('['): cmd += ['--mon-addrv', mons[first_mon_role]] else: @@ -421,7 +412,7 @@ def ceph_bootstrap(ctx, config): for remote in ctx.cluster.remotes.keys(): if remote == bootstrap_remote: continue - log.info('Writing conf and keyring to %s' % remote.shortname) + log.info('Writing (initial) conf and keyring to %s' % remote.shortname) teuthology.write_file( remote=remote, path='/etc/ceph/{}.conf'.format(cluster_name), @@ -516,19 +507,18 @@ def ceph_mons(ctx, config): if len(j['mons']) == num_mons: break - # refresh ceph.conf files for all mons + first mgr - for remote, roles in ctx.cluster.remotes.items(): - for mon in [r for r in roles - if teuthology.is_type('mon', cluster_name)(r)]: - c_, _, id_ = teuthology.split_role(mon) - _shell(ctx, cluster_name, remote, [ - 'ceph', 'orch', 'daemon', 'reconfig', - 'mon.' + id_, - ]) - _shell(ctx, cluster_name, ctx.ceph[cluster_name].bootstrap_remote, [ - 'ceph', 'orch', 'daemon', 'reconfig', - 'mgr.' + ctx.ceph[cluster_name].first_mgr, - ]) + # refresh our (final) ceph.conf file + log.info('Generating final ceph.conf file...') + r = _shell( + ctx=ctx, + cluster_name=cluster_name, + remote=remote, + args=[ + 'ceph', 'config', 'generate-minimal-conf', + ], + stdout=BytesIO(), + ) + ctx.ceph[cluster_name].config_file = r.stdout.getvalue() yield @@ -583,6 +573,7 @@ def ceph_osds(ctx, config): """ cluster_name = config['cluster'] fsid = ctx.ceph[cluster_name].fsid + try: log.info('Deploying OSDs...') @@ -931,11 +922,34 @@ def restart(ctx, config): ctx.managers[cluster].wait_for_all_osds_up() yield +@contextlib.contextmanager +def distribute_config_and_admin_keyring(ctx, config): + """ + Distribute a sufficient config and keyring for clients + """ + cluster_name = config['cluster'] + log.info('Distributing (final) config and client.admin keyring...') + for remote, roles in ctx.cluster.remotes.items(): + teuthology.sudo_write_file( + remote=remote, + path='/etc/ceph/{}.conf'.format(cluster_name), + data=ctx.ceph[cluster_name].config_file) + teuthology.sudo_write_file( + remote=remote, + path='/etc/ceph/{}.client.admin.keyring'.format(cluster_name), + data=ctx.ceph[cluster_name].admin_keyring) + try: + yield + finally: + ctx.cluster.run(args=[ + 'sudo', 'rm', '-f', + '/etc/ceph/{}.conf'.format(cluster_name), + '/etc/ceph/{}.client.admin.keyring'.format(cluster_name), + ]) + @contextlib.contextmanager def crush_setup(ctx, config): cluster_name = config['cluster'] - first_mon = teuthology.get_first_mon(ctx, config, cluster_name) - (mon_remote,) = ctx.cluster.only(first_mon).remotes.keys() profile = config.get('crush_tunables', 'default') log.info('Setting crush tunables to %s', profile) @@ -972,6 +986,8 @@ def task(ctx, config): ctx.ceph[cluster_name].thrashers = [] # fixme: setup watchdog, ala ceph.py + ctx.ceph[cluster_name].roleless = False # see below + # cephadm mode? if 'cephadm_mode' not in config: config['cephadm_mode'] = 'root' @@ -1012,13 +1028,51 @@ def task(ctx, config): roles = [role_list for (remote, role_list) in remotes_and_roles] ips = [host for (host, port) in (remote.ssh.get_transport().getpeername() for (remote, role_list) in remotes_and_roles)] + + if config.get('roleless', False): + # mons will be named after hosts + n = len(roles) + roles = [] + first_mon = None + for remote, _ in remotes_and_roles: + roles.append(['mon.' + remote.shortname]) + if not first_mon: + first_mon = remote.shortname + bootstrap_remote = remote + log.info('No roles; fabricating mons %s' % roles) + ctx.ceph[cluster_name].mons = get_mons( roles, ips, cluster_name, mon_bind_msgr2=config.get('mon_bind_msgr2', True), mon_bind_addrvec=config.get('mon_bind_addrvec', True), - ) + ) log.info('Monitor IPs: %s' % ctx.ceph[cluster_name].mons) + if config.get('roleless', False): + ctx.ceph[cluster_name].roleless = True + ctx.ceph[cluster_name].bootstrap_remote = bootstrap_remote + ctx.ceph[cluster_name].first_mon = first_mon + ctx.ceph[cluster_name].first_mon_role = 'mon.' + first_mon + else: + first_mon_role = sorted(ctx.ceph[cluster_name].mons.keys())[0] + _, _, first_mon = teuthology.split_role(first_mon_role) + (bootstrap_remote,) = ctx.cluster.only(first_mon_role).remotes.keys() + log.info('First mon is mon.%s on %s' % (first_mon, + bootstrap_remote.shortname)) + ctx.ceph[cluster_name].bootstrap_remote = bootstrap_remote + ctx.ceph[cluster_name].first_mon = first_mon + ctx.ceph[cluster_name].first_mon_role = first_mon_role + + others = ctx.cluster.remotes[bootstrap_remote] + mgrs = sorted([r for r in others + if teuthology.is_type('mgr', cluster_name)(r)]) + if not mgrs: + raise RuntimeError('no mgrs on the same host as first mon %s' % first_mon) + _, _, first_mgr = teuthology.split_role(mgrs[0]) + log.info('First mgr is %s' % (first_mgr)) + ctx.ceph[cluster_name].first_mgr = first_mgr + + with contextutil.nested( lambda: ceph_initial(), lambda: normalize_hostnames(ctx=ctx), @@ -1028,6 +1082,7 @@ def task(ctx, config): lambda: ceph_bootstrap(ctx=ctx, config=config), lambda: crush_setup(ctx=ctx, config=config), lambda: ceph_mons(ctx=ctx, config=config), + lambda: distribute_config_and_admin_keyring(ctx=ctx, config=config), lambda: ceph_mgrs(ctx=ctx, config=config), lambda: ceph_osds(ctx=ctx, config=config), lambda: ceph_mdss(ctx=ctx, config=config), diff --git a/ceph/qa/tasks/cephfs/fuse_mount.py b/ceph/qa/tasks/cephfs/fuse_mount.py index 21a9a6d2f..6ca08a460 100644 --- a/ceph/qa/tasks/cephfs/fuse_mount.py +++ b/ceph/qa/tasks/cephfs/fuse_mount.py @@ -244,6 +244,9 @@ class FuseMount(CephFSMount): return self.client_remote.run(args=["ls", "-d", self.mountpoint], check_status=False, cwd=self.test_dir, timeout=(15*60)).exitstatus == 0 def umount(self): + if not self.is_mounted(): + return + try: log.info('Running fusermount -u on {name}...'.format(name=self.client_remote.name)) self.client_remote.run( diff --git a/ceph/qa/tasks/cephfs/kernel_mount.py b/ceph/qa/tasks/cephfs/kernel_mount.py index 86a06006d..8e4eeb66a 100644 --- a/ceph/qa/tasks/cephfs/kernel_mount.py +++ b/ceph/qa/tasks/cephfs/kernel_mount.py @@ -74,6 +74,9 @@ class KernelMount(CephFSMount): self.mounted = True def umount(self, force=False): + if not self.is_mounted(): + return + log.debug('Unmounting client client.{id}...'.format(id=self.client_id)) cmd=['sudo', 'umount', self.mountpoint] diff --git a/ceph/qa/tasks/cephfs/test_admin.py b/ceph/qa/tasks/cephfs/test_admin.py index 374f75a72..c53548960 100644 --- a/ceph/qa/tasks/cephfs/test_admin.py +++ b/ceph/qa/tasks/cephfs/test_admin.py @@ -1,3 +1,5 @@ +import json + from teuthology.orchestra.run import CommandFailedError from tasks.cephfs.cephfs_test_case import CephFSTestCase @@ -30,6 +32,11 @@ class TestAdminCommands(CephFSTestCase): if overwrites: self.fs.mon_manager.raw_cluster_cmd('osd', 'pool', 'set', n+"-data", 'allow_ec_overwrites', 'true') + def _check_pool_application_metadata_key_value(self, pool, app, key, value): + output = self.fs.mon_manager.raw_cluster_cmd( + 'osd', 'pool', 'application', 'get', pool, app, key) + self.assertEqual(str(output.strip()), value) + def test_add_data_pool_root(self): """ That a new data pool can be added and used for the root directory. @@ -38,6 +45,19 @@ class TestAdminCommands(CephFSTestCase): p = self.fs.add_data_pool("foo") self.fs.set_dir_layout(self.mount_a, ".", FileLayout(pool=p)) + def test_add_data_pool_application_metadata(self): + """ + That the application metadata set on a newly added data pool is as expected. + """ + pool_name = "foo" + mon_cmd = self.fs.mon_manager.raw_cluster_cmd + mon_cmd('osd', 'pool', 'create', pool_name, str(self.fs.pgs_per_fs_pool)) + # Check whether https://tracker.ceph.com/issues/43061 is fixed + mon_cmd('osd', 'pool', 'application', 'enable', pool_name, 'cephfs') + self.fs.add_data_pool(pool_name, create=False) + self._check_pool_application_metadata_key_value( + pool_name, 'cephfs', 'data', self.fs.name) + def test_add_data_pool_subdir(self): """ That a new data pool can be added and used for a sub-directory. @@ -112,6 +132,24 @@ class TestAdminCommands(CephFSTestCase): else: raise RuntimeError("expected failure") + def test_fs_new_pool_application_metadata(self): + """ + That the application metadata set on the pools of a newly created filesystem are as expected. + """ + self.fs.delete_all_filesystems() + fs_name = "test_fs_new_pool_application" + keys = ['metadata', 'data'] + pool_names = [fs_name+'-'+key for key in keys] + mon_cmd = self.fs.mon_manager.raw_cluster_cmd + for p in pool_names: + mon_cmd('osd', 'pool', 'create', p, str(self.fs.pgs_per_fs_pool)) + mon_cmd('osd', 'pool', 'application', 'enable', p, 'cephfs') + mon_cmd('fs', 'new', fs_name, pool_names[0], pool_names[1]) + for i in range(2): + self._check_pool_application_metadata_key_value( + pool_names[i], 'cephfs', keys[i], fs_name) + + class TestConfigCommands(CephFSTestCase): """ Test that daemons and clients respond to the otherwise rarely-used diff --git a/ceph/qa/tasks/cephfs/test_data_scan.py b/ceph/qa/tasks/cephfs/test_data_scan.py index fe099838e..512e47541 100644 --- a/ceph/qa/tasks/cephfs/test_data_scan.py +++ b/ceph/qa/tasks/cephfs/test_data_scan.py @@ -429,6 +429,10 @@ class TestDataScan(CephFSTestCase): file_count = 100 file_names = ["%s" % n for n in range(0, file_count)] + # Make sure and disable dirfrag auto merging and splitting + self.fs.set_ceph_conf('mds', 'mds bal merge size', 0) + self.fs.set_ceph_conf('mds', 'mds bal split size', 100 * file_count) + # Create a directory of `file_count` files, each named after its # decimal number and containing the string of its decimal number self.mount_a.run_python(dedent(""" @@ -483,6 +487,7 @@ class TestDataScan(CephFSTestCase): # by checking the omap now has the dentry's key again self.fs.data_scan(["scan_extents", self.fs.get_data_pool_name()]) self.fs.data_scan(["scan_inodes", self.fs.get_data_pool_name()]) + self.fs.data_scan(["scan_links"]) self.assertIn(victim_key, self._dirfrag_keys(frag_obj_id)) # Start the filesystem and check that the dentry we deleted is now once again visible @@ -503,6 +508,14 @@ class TestDataScan(CephFSTestCase): keys = self._dirfrag_keys(frag_obj_id) self.assertListEqual(sorted(keys), sorted(["%s_head" % f for f in file_names])) + # run scrub to update and make sure rstat.rbytes info in subdir inode and dirfrag + # are matched + out_json = self.fs.rank_tell(["scrub", "start", "/subdir", "repair", "recursive"]) + self.assertNotEqual(out_json, None) + + # Remove the whole 'sudbdir' directory + self.mount_a.run_shell(["rm", "-rf", "subdir/"]) + @for_teuthology def test_parallel_execution(self): self._rebuild_metadata(ManyFilesWorkload(self.fs, self.mount_a, 25), workers=7) diff --git a/ceph/qa/tasks/cephfs/test_full.py b/ceph/qa/tasks/cephfs/test_full.py index 112407de1..3ba05af1d 100644 --- a/ceph/qa/tasks/cephfs/test_full.py +++ b/ceph/qa/tasks/cephfs/test_full.py @@ -45,18 +45,26 @@ class FullnessTestCase(CephFSTestCase): epoch. """ - # Sync up clients with initial MDS OSD map barrier - self.mount_a.open_no_data("foo") - self.mount_b.open_no_data("bar") + # script that sync up client with MDS OSD map barrier. The barrier should + # be updated by cap flush ack message. + pyscript = dedent(""" + import os + fd = os.open("{path}", os.O_CREAT | os.O_RDWR, 0O600) + os.fchmod(fd, 0O666) + os.fsync(fd) + os.close(fd) + """) + + # Sync up client with initial MDS OSD map barrier. + path = os.path.join(self.mount_a.mountpoint, "foo") + self.mount_a.run_python(pyscript.format(path=path)) # Grab mounts' initial OSD epochs: later we will check that # it hasn't advanced beyond this point. - mount_a_initial_epoch = self.mount_a.get_osd_epoch()[0] - mount_b_initial_epoch = self.mount_b.get_osd_epoch()[0] + mount_a_initial_epoch, mount_a_initial_barrier = self.mount_a.get_osd_epoch() # Freshly mounted at start of test, should be up to date with OSD map self.assertGreaterEqual(mount_a_initial_epoch, self.initial_osd_epoch) - self.assertGreaterEqual(mount_b_initial_epoch, self.initial_osd_epoch) # Set and unset a flag to cause OSD epoch to increment self.fs.mon_manager.raw_cluster_cmd("osd", "set", "pause") @@ -69,43 +77,28 @@ class FullnessTestCase(CephFSTestCase): # Do a metadata operation on clients, witness that they end up with # the old OSD map from startup time (nothing has prompted client # to update its map) - self.mount_a.open_no_data("alpha") - self.mount_b.open_no_data("bravo1") - - # Sleep long enough that if the OSD map was propagating it would - # have done so (this is arbitrary because we are 'waiting' for something - # to *not* happen). - time.sleep(30) - + path = os.path.join(self.mount_a.mountpoint, "foo") + self.mount_a.run_python(pyscript.format(path=path)) mount_a_epoch, mount_a_barrier = self.mount_a.get_osd_epoch() self.assertEqual(mount_a_epoch, mount_a_initial_epoch) - mount_b_epoch, mount_b_barrier = self.mount_b.get_osd_epoch() - self.assertEqual(mount_b_epoch, mount_b_initial_epoch) + self.assertEqual(mount_a_barrier, mount_a_initial_barrier) # Set a barrier on the MDS self.fs.rank_asok(["osdmap", "barrier", new_epoch.__str__()]) - # Do an operation on client B, witness that it ends up with - # the latest OSD map from the barrier. This shouldn't generate any - # cap revokes to A because B was already the last one to touch - # a file in root. - self.mount_b.run_shell(["touch", "bravo2"]) - self.mount_b.open_no_data("bravo2") + # Sync up client with new MDS OSD map barrier + path = os.path.join(self.mount_a.mountpoint, "baz") + self.mount_a.run_python(pyscript.format(path=path)) + mount_a_epoch, mount_a_barrier = self.mount_a.get_osd_epoch() + self.assertEqual(mount_a_barrier, new_epoch) # Some time passes here because the metadata part of the operation # completes immediately, while the resulting OSD map update happens # asynchronously (it's an Objecter::_maybe_request_map) as a result # of seeing the new epoch barrier. - self.wait_until_equal( - lambda: self.mount_b.get_osd_epoch(), - (new_epoch, new_epoch), - 30, - lambda x: x[0] > new_epoch or x[1] > new_epoch) - - # ...and none of this should have affected the oblivious mount a, - # because it wasn't doing any data or metadata IO - mount_a_epoch, mount_a_barrier = self.mount_a.get_osd_epoch() - self.assertEqual(mount_a_epoch, mount_a_initial_epoch) + self.wait_until_true( + lambda: self.mount_a.get_osd_epoch()[0] >= new_epoch, + timeout=30) def _data_pool_name(self): data_pool_names = self.fs.get_data_pool_names() diff --git a/ceph/qa/tasks/cephfs/test_scrub_checks.py b/ceph/qa/tasks/cephfs/test_scrub_checks.py index e3f5609af..012b6c009 100644 --- a/ceph/qa/tasks/cephfs/test_scrub_checks.py +++ b/ceph/qa/tasks/cephfs/test_scrub_checks.py @@ -16,7 +16,7 @@ class TestScrubControls(CephFSTestCase): Test basic scrub control operations such as abort, pause and resume. """ - MDSS_REQUIRED = 1 + MDSS_REQUIRED = 2 CLIENTS_REQUIRED = 1 def _abort_scrub(self, expected): @@ -129,6 +129,34 @@ class TestScrubControls(CephFSTestCase): time.sleep(10) self._check_task_status("idle") + def test_scrub_task_status_on_mds_failover(self): + # sleep enough to fetch updated task status + time.sleep(10) + + (original_active, ) = self.fs.get_active_names() + original_standbys = self.mds_cluster.get_standby_daemons() + self._check_task_status("idle") + + # Kill the rank 0 + self.fs.mds_stop(original_active) + + grace = float(self.fs.get_config("mds_beacon_grace", service_type="mon")) + + def promoted(): + active = self.fs.get_active_names() + return active and active[0] in original_standbys + + log.info("Waiting for promotion of one of the original standbys {0}".format( + original_standbys)) + self.wait_until_true(promoted, timeout=grace*2) + + mgr_beacon_grace = float(self.fs.get_config("mgr_service_beacon_grace", service_type="mon")) + + def status_check(): + task_status = self.fs.get_task_status("scrub status") + return original_active not in task_status + self.wait_until_true(status_check, timeout=mgr_beacon_grace*2) + class TestScrubChecks(CephFSTestCase): """ Run flush and scrub commands on the specified files in the filesystem. This diff --git a/ceph/qa/tasks/cephfs/test_volumes.py b/ceph/qa/tasks/cephfs/test_volumes.py index 6f5bdec23..9fca6de56 100644 --- a/ceph/qa/tasks/cephfs/test_volumes.py +++ b/ceph/qa/tasks/cephfs/test_volumes.py @@ -156,6 +156,14 @@ class TestVolumes(CephFSTestCase): # remove the leading '/', and trailing whitespaces return path[1:].rstrip() + def _get_subvolume_info(self, vol_name, subvol_name, group_name=None): + args = ["subvolume", "info", vol_name, subvol_name] + if group_name: + args.append(group_name) + args = tuple(args) + subvol_md = self._fs_cmd(*args) + return subvol_md + def _delete_test_volume(self): self._fs_cmd("volume", "rm", self.volname, "--yes-i-really-mean-it") @@ -733,6 +741,106 @@ class TestVolumes(CephFSTestCase): raise RuntimeError("expected filling subvolume {0} with {1} file of size {2}MB " "to succeed".format(subvolname, number_of_files, file_size)) + def test_subvolume_info(self): + # tests the 'fs subvolume info' command + + subvol_md = ["atime", "bytes_pcent", "bytes_quota", "bytes_used", "created_at", "ctime", + "data_pool", "gid", "mode", "mon_addrs", "mtime", "path", "type", "uid"] + + # create subvolume + subvolume = self._generate_random_subvolume_name() + self._fs_cmd("subvolume", "create", self.volname, subvolume) + + # get subvolume metadata + subvol_info = json.loads(self._get_subvolume_info(self.volname, subvolume)) + if len(subvol_info) == 0: + raise RuntimeError("Expected the 'fs subvolume info' command to list metadata of subvolume") + for md in subvol_md: + if md not in subvol_info.keys(): + raise RuntimeError("%s not present in the metadata of subvolume" % md) + + if subvol_info["bytes_pcent"] != "undefined": + raise RuntimeError("bytes_pcent should be set to undefined if quota is not set") + + if subvol_info["bytes_quota"] != "infinite": + raise RuntimeError("bytes_quota should be set to infinite if quota is not set") + + nsize = self.DEFAULT_FILE_SIZE*1024*1024 + try: + self._fs_cmd("subvolume", "resize", self.volname, subvolume, str(nsize)) + except CommandFailedError: + raise RuntimeError("expected the 'fs subvolume resize' command to succeed") + + # get subvolume metadata after quota set + subvol_info = json.loads(self._get_subvolume_info(self.volname, subvolume)) + if len(subvol_info) == 0: + raise RuntimeError("Expected the 'fs subvolume info' command to list metadata of subvolume") + if subvol_info["bytes_pcent"] == "undefined": + raise RuntimeError("bytes_pcent should not be set to undefined if quota is set") + + if subvol_info["bytes_quota"] == "infinite": + raise RuntimeError("bytes_quota should not be set to infinite if quota is set") + + if subvol_info["type"] != "subvolume": + raise RuntimeError("type should be set to subvolume") + + # remove subvolumes + self._fs_cmd("subvolume", "rm", self.volname, subvolume) + + # verify trash dir is clean + self._wait_for_trash_empty() + + def test_clone_subvolume_info(self): + + # tests the 'fs subvolume info' command for a clone + subvol_md = ["atime", "bytes_pcent", "bytes_quota", "bytes_used", "created_at", "ctime", + "data_pool", "gid", "mode", "mon_addrs", "mtime", "path", "type", "uid"] + + subvolume = self._generate_random_subvolume_name() + snapshot = self._generate_random_snapshot_name() + clone = self._generate_random_clone_name() + + # create subvolume + self._fs_cmd("subvolume", "create", self.volname, subvolume) + + # do some IO + self._do_subvolume_io(subvolume, number_of_files=1) + + # snapshot subvolume + self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) + + # now, protect snapshot + self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) + + # schedule a clone + self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone) + + # check clone status + self._wait_for_clone_to_complete(clone) + + # now, unprotect snapshot + self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot) + + # remove snapshot + self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) + + subvol_info = json.loads(self._get_subvolume_info(self.volname, clone)) + if len(subvol_info) == 0: + raise RuntimeError("Expected the 'fs subvolume info' command to list metadata of subvolume") + for md in subvol_md: + if md not in subvol_info.keys(): + raise RuntimeError("%s not present in the metadata of subvolume" % md) + if subvol_info["type"] != "clone": + raise RuntimeError("type should be set to clone") + + # remove subvolumes + self._fs_cmd("subvolume", "rm", self.volname, subvolume) + self._fs_cmd("subvolume", "rm", self.volname, clone) + + # verify trash dir is clean + self._wait_for_trash_empty() + + ### subvolume group operations def test_subvolume_create_and_rm_in_group(self): diff --git a/ceph/qa/tasks/cephfs/xfstests_dev.py b/ceph/qa/tasks/cephfs/xfstests_dev.py index 15ee52ca5..c11561fae 100644 --- a/ceph/qa/tasks/cephfs/xfstests_dev.py +++ b/ceph/qa/tasks/cephfs/xfstests_dev.py @@ -15,7 +15,7 @@ logger = logging.getLogger(__name__) class XFSTestsDev(CephFSTestCase): def setUp(self): - CephFSTestCase.setUp(self) + super(XFSTestsDev, self).setUp() self.prepare_xfstests_dev() def prepare_xfstests_dev(self): @@ -172,3 +172,5 @@ class XFSTestsDev(CephFSTestCase): self.mount_a.client_remote.run(args=['sudo', 'rm', '-rf', self.repo_path], omit_sudo=False, check_status=False) + + super(XFSTestsDev, self).tearDown() diff --git a/ceph/qa/tasks/mgr/dashboard/test_perf_counters.py b/ceph/qa/tasks/mgr/dashboard/test_perf_counters.py index c3191dc5e..c01368bce 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_perf_counters.py +++ b/ceph/qa/tasks/mgr/dashboard/test_perf_counters.py @@ -39,7 +39,7 @@ class PerfCountersControllerTest(DashboardTestCase): self._validate_perf(mon, 'mon', data, allow_empty=False) def test_perf_counters_mgr_get(self): - mgr = self.mgr_cluster.mgr_ids[0] + mgr = list(self.mgr_cluster.mgr_ids)[0] data = self._get('/api/perf_counters/mgr/{}'.format(mgr)) self.assertStatus(200) self._validate_perf(mgr, 'mgr', data, allow_empty=False) diff --git a/ceph/qa/tasks/mgr/dashboard/test_rgw.py b/ceph/qa/tasks/mgr/dashboard/test_rgw.py index 9cb3504b5..47a6de245 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_rgw.py +++ b/ceph/qa/tasks/mgr/dashboard/test_rgw.py @@ -116,7 +116,7 @@ class RgwBucketTest(RgwTestCase): _mfa_token_serial = '1' _mfa_token_seed = '23456723' - _mfa_token_time_step = 3 + _mfa_token_time_step = 2 AUTH_ROLES = ['rgw-manager'] @@ -152,7 +152,7 @@ class RgwBucketTest(RgwTestCase): totp_key = base64.b32decode(self._mfa_token_seed) totp = TOTP(totp_key, 6, SHA1(), self._mfa_token_time_step, backend=default_backend(), enforce_key_length=False) - time_value = time.time() + time_value = int(time.time()) return totp.generate(time_value) def test_all(self): @@ -241,7 +241,7 @@ class RgwBucketTest(RgwTestCase): self.assertEqual(data['mfa_delete'], 'Enabled') # Update bucket: disable versioning & MFA Delete. - time.sleep(self._mfa_token_time_step + 2) # Required to get new TOTP pin. + time.sleep(self._mfa_token_time_step * 3) # Required to get new TOTP pin. self._put( '/api/rgw/bucket/teuth-test-bucket', params={ diff --git a/ceph/qa/tasks/radosbench.py b/ceph/qa/tasks/radosbench.py index f840e646d..90b21a5d8 100644 --- a/ceph/qa/tasks/radosbench.py +++ b/ceph/qa/tasks/radosbench.py @@ -23,6 +23,7 @@ def task(ctx, config): time: pool: size: write size to use + concurrency: max number of outstanding writes (16) objectsize: object size to use unique_pool: use a unique pool, defaults to False ec_pool: create an ec pool, defaults to False @@ -83,6 +84,7 @@ def task(ctx, config): pool = manager.create_pool_with_unique_name(erasure_code_profile_name=profile_name) size = config.get('size', 65536) + concurrency = config.get('concurrency', 16) osize = config.get('objectsize', 65536) sizeargs = ['-b', str(size)] if osize != 0 and osize != size: @@ -102,6 +104,7 @@ def task(ctx, config): '--no-log-to-stderr', '--name', role] + sizeargs + + ['-t', str(concurrency)] + ['-p' , pool, 'bench', str(60), "write", "--no-cleanup" ]).format(tdir=testdir), diff --git a/ceph/qa/tasks/rgw.py b/ceph/qa/tasks/rgw.py index df900d2c4..2d99d9962 100644 --- a/ceph/qa/tasks/rgw.py +++ b/ceph/qa/tasks/rgw.py @@ -132,10 +132,11 @@ def start_rgw(ctx, config, clients): raise ConfigError('vault: no "root_token" specified') # create token on file ctx.cluster.only(client).run(args=['echo', '-n', ctx.vault.root_token, run.Raw('>'), token_path]) - log.info("Restrict access to token file") - ctx.cluster.only(client).run(args=['chmod', '600', token_path]) log.info("Token file content") ctx.cluster.only(client).run(args=['cat', token_path]) + log.info("Restrict access to token file") + ctx.cluster.only(client).run(args=['chmod', '600', token_path]) + ctx.cluster.only(client).run(args=['sudo', 'chown', 'ceph', token_path]) rgw_cmd.extend([ '--rgw_crypt_vault_addr', "{}:{}".format(*ctx.vault.endpoints[vault_role]), diff --git a/ceph/qa/tasks/vstart_runner.py b/ceph/qa/tasks/vstart_runner.py index 7055ebb7c..df7005e99 100644 --- a/ceph/qa/tasks/vstart_runner.py +++ b/ceph/qa/tasks/vstart_runner.py @@ -30,6 +30,7 @@ Alternative usage: """ +from six import StringIO from io import BytesIO from collections import defaultdict import getpass @@ -150,6 +151,11 @@ else: SRC_PREFIX = "./" +def rm_nonascii_chars(var): + var = var.replace('\xe2\x80\x98', '\'') + var = var.replace('\xe2\x80\x99', '\'') + return var + class LocalRemoteProcess(object): def __init__(self, args, subproc, check_status, stdout, stderr): self.args = args @@ -170,6 +176,7 @@ class LocalRemoteProcess(object): return out, err = self.subproc.communicate() + out, err = rm_nonascii_chars(out), rm_nonascii_chars(err) self.stdout.write(out) self.stderr.write(err) @@ -333,7 +340,7 @@ class LocalRemote(object): # Filter out helper tools that don't exist in a vstart environment args = [a for a in args if a not in { - 'adjust-ulimits', 'ceph-coverage', 'timeout'}] + 'adjust-ulimits', 'ceph-coverage'}] # Adjust binary path prefix if given a bare program name if "/" not in args[0]: @@ -615,33 +622,6 @@ class LocalKernelMount(KernelMount): log.info("I think my launching pid was {0}".format(self.fuse_daemon.subproc.pid)) return path - def umount(self, force=False): - log.debug('Unmounting client client.{id}...'.format(id=self.client_id)) - - cmd=['sudo', 'umount', self.mountpoint] - if force: - cmd.append('-f') - - try: - self.client_remote.run(args=cmd, timeout=(15*60), omit_sudo=False) - except Exception as e: - self.client_remote.run(args=[ - 'sudo', - Raw('PATH=/usr/sbin:$PATH'), - 'lsof', - Raw(';'), - 'ps', 'auxf', - ], timeout=(15*60), omit_sudo=False) - raise e - - rproc = self.client_remote.run(args=[ - 'rmdir', - '--', - self.mountpoint, - ]) - rproc.wait() - self.mounted = False - def mount(self, mount_path=None, mount_fs_name=None, mount_options=[]): self.setupfs(name=mount_fs_name) @@ -803,10 +783,6 @@ class LocalFuseMount(FuseMount): log.info("I think my launching pid was {0}".format(self.fuse_daemon.subproc.pid)) return path - def umount(self): - if self.is_mounted(): - super(LocalFuseMount, self).umount() - def mount(self, mount_path=None, mount_fs_name=None, mountpoint=None, mount_options=[]): if mountpoint is not None: self.mountpoint = mountpoint @@ -935,7 +911,7 @@ class LocalCephManager(CephManager): if watch_channel is not None: args.append("--watch-channel") args.append(watch_channel) - proc = self.controller.run(args=args, wait=False, stdout=BytesIO()) + proc = self.controller.run(args=args, wait=False, stdout=StringIO()) return proc def raw_cluster_cmd(self, *args, **kwargs): diff --git a/ceph/qa/workunits/cephadm/test_adoption.sh b/ceph/qa/workunits/cephadm/test_adoption.sh index 7960ede00..83e11a40a 100755 --- a/ceph/qa/workunits/cephadm/test_adoption.sh +++ b/ceph/qa/workunits/cephadm/test_adoption.sh @@ -3,6 +3,7 @@ SCRIPT_NAME=$(basename ${BASH_SOURCE[0]}) SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" CEPHADM_SRC_DIR=${SCRIPT_DIR}/../../../src/cephadm +CORPUS_COMMIT=50c5dd734638939facd1ed32295ce59c9a5986b4 [ -z "$SUDO" ] && SUDO=sudo if [ -z "$CEPHADM" ]; then @@ -54,9 +55,11 @@ CEPHADM="$SUDO $CEPHADM_BIN" CORPUS_GIT_SUBMOD="cephadm-adoption-corpus" TMPDIR=$(mktemp -d) git clone https://github.com/ceph/$CORPUS_GIT_SUBMOD $TMPDIR -CORPUS_DIR=${TMPDIR}/archive trap "$SUDO rm -rf $TMPDIR" EXIT +git -C $TMPDIR checkout $CORPUS_COMMIT +CORPUS_DIR=${TMPDIR}/archive + for subdir in `ls ${CORPUS_DIR}`; do for tarfile in `ls ${CORPUS_DIR}/${subdir} | grep .tgz`; do tarball=${CORPUS_DIR}/${subdir}/${tarfile} diff --git a/ceph/qa/workunits/cephadm/test_cephadm.sh b/ceph/qa/workunits/cephadm/test_cephadm.sh index 1be330d80..42f195568 100755 --- a/ceph/qa/workunits/cephadm/test_cephadm.sh +++ b/ceph/qa/workunits/cephadm/test_cephadm.sh @@ -3,22 +3,16 @@ SCRIPT_NAME=$(basename ${BASH_SOURCE[0]}) SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# cleanup during exit +[ -z "$CLEANUP" ] && CLEANUP=true + FSID='00000000-0000-0000-0000-0000deadbeef' # images that are used -IMAGE_MASTER=${IMAGE_MASTER:-'quay.io/ceph-ci/ceph:octopus'} # octopus for octopus branch +IMAGE_MASTER=${IMAGE_MASTER:-'docker.io/ceph/daemon-base:latest-octopus'} IMAGE_NAUTILUS=${IMAGE_NAUTILUS:-'docker.io/ceph/daemon-base:latest-nautilus'} IMAGE_MIMIC=${IMAGE_MIMIC:-'docker.io/ceph/daemon-base:latest-mimic'} -TMPDIR=$(mktemp -d) - -function cleanup() -{ - dump_all_logs - rm -rf $TMPDIR -} -trap cleanup EXIT - OSD_IMAGE_NAME="${SCRIPT_NAME%.*}_osd.img" OSD_IMAGE_SIZE='6G' OSD_TO_CREATE=2 @@ -86,6 +80,26 @@ if ! [ "$loopdev" = "" ]; then $SUDO losetup -d $loopdev fi +# TMPDIR for test data +[ -d "$TMPDIR" ] || TMPDIR=$(mktemp -d tmp.$SCRIPT_NAME.XXXXXX) + +function cleanup() +{ + if [ $CLEANUP = false ]; then + # preserve the TMPDIR state + echo "========================" + echo "!!! CLEANUP=$CLEANUP !!!" + echo + echo "TMPDIR=$TMPDIR" + echo "========================" + return + fi + + dump_all_logs $FSID + rm -rf $TMPDIR +} +trap cleanup EXIT + function expect_false() { set -x @@ -114,8 +128,9 @@ function is_available() function dump_log() { - local name="$1" - local num_lines="$2" + local fsid="$1" + local name="$2" + local num_lines="$3" if [ -z $num_lines ]; then num_lines=100 @@ -125,16 +140,17 @@ function dump_log() echo 'dump daemon log:' $name echo '-------------------------' - $CEPHADM logs --name $name -- --no-pager -n $num_lines + $CEPHADM logs --fsid $fsid --name $name -- --no-pager -n $num_lines } function dump_all_logs() { - names=$($CEPHADM ls | jq -r '.[].name') + local fsid="$1" + local names=$($CEPHADM ls | jq -r '.[] | select(.fsid == "'$fsid'").name') echo 'dumping logs for daemons: ' $names for name in $names; do - dump_log $name + dump_log $fsid $name done } @@ -288,16 +304,17 @@ done for id in `seq 0 $((--OSD_TO_CREATE))`; do device_name=/dev/$OSD_VG_NAME/$OSD_LV_NAME.$id + CEPH_VOLUME="$CEPHADM ceph-volume \ + --fsid $FSID \ + --config $CONFIG \ + --keyring $TMPDIR/keyring.bootstrap.osd --" # prepare the osd - $CEPHADM ceph-volume --config $CONFIG --keyring $TMPDIR/keyring.bootstrap.osd -- \ - lvm prepare --bluestore --data $device_name --no-systemd - $CEPHADM ceph-volume --config $CONFIG --keyring $TMPDIR/keyring.bootstrap.osd -- \ - lvm batch --no-auto $device_name --yes --no-systemd + $CEPH_VOLUME lvm prepare --bluestore --data $device_name --no-systemd + $CEPH_VOLUME lvm batch --no-auto $device_name --yes --no-systemd # osd id and osd fsid - $CEPHADM ceph-volume --config $CONFIG --keyring $TMPDIR/keyring.bootstrap.osd -- \ - lvm list --format json $device_name > $TMPDIR/osd.map + $CEPH_VOLUME lvm list --format json $device_name > $TMPDIR/osd.map osd_id=$($SUDO cat $TMPDIR/osd.map | jq -cr '.. | ."ceph.osd_id"? | select(.)') osd_fsid=$($SUDO cat $TMPDIR/osd.map | jq -cr '.. | ."ceph.osd_fsid"? | select(.)') @@ -363,8 +380,8 @@ $CEPHADM unit --fsid $FSID --name mon.a -- is-enabled ## shell $CEPHADM shell --fsid $FSID -- true $CEPHADM shell --fsid $FSID -- test -d /var/log/ceph -expect_false $CEPHADM --timeout 1 shell --fsid $FSID -- sleep 10 -$CEPHADM --timeout 10 shell --fsid $FSID -- sleep 1 +expect_false $CEPHADM --timeout 10 shell --fsid $FSID -- sleep 60 +$CEPHADM --timeout 60 shell --fsid $FSID -- sleep 10 ## enter expect_false $CEPHADM enter @@ -374,13 +391,16 @@ $CEPHADM enter --fsid $FSID --name mon.a -- pidof ceph-mon expect_false $CEPHADM enter --fsid $FSID --name mgr.x -- pidof ceph-mon $CEPHADM enter --fsid $FSID --name mgr.x -- pidof ceph-mgr # this triggers a bug in older versions of podman, including 18.04's 1.6.2 -#expect_false $CEPHADM --timeout 1 enter --fsid $FSID --name mon.a -- sleep 10 -$CEPHADM --timeout 10 enter --fsid $FSID --name mon.a -- sleep 1 +#expect_false $CEPHADM --timeout 5 enter --fsid $FSID --name mon.a -- sleep 30 +$CEPHADM --timeout 60 enter --fsid $FSID --name mon.a -- sleep 10 ## ceph-volume $CEPHADM ceph-volume --fsid $FSID -- inventory --format=json \ | jq '.[]' +## preserve test state +[ $CLEANUP = false ] && exit 0 + ## rm-daemon # mon and osd require --force expect_false $CEPHADM rm-daemon --fsid $FSID --name mon.a @@ -391,5 +411,4 @@ $CEPHADM rm-daemon --fsid $FSID --name mgr.x expect_false $CEPHADM rm-cluster --fsid $FSID $CEPHADM rm-cluster --fsid $FSID --force -rm -rf $TMPDIR echo PASS diff --git a/ceph/qa/workunits/mon/pool_ops.sh b/ceph/qa/workunits/mon/pool_ops.sh index 4098795b9..b02077691 100755 --- a/ceph/qa/workunits/mon/pool_ops.sh +++ b/ceph/qa/workunits/mon/pool_ops.sh @@ -8,6 +8,41 @@ function expect_false() if "$@"; then return 1; else return 0; fi } +function get_config_value_or_die() +{ + local pool_name config_opt raw val + + pool_name=$1 + config_opt=$2 + + raw="`$SUDO ceph osd pool get $pool_name $config_opt 2>/dev/null`" + if [[ $? -ne 0 ]]; then + echo "error obtaining config opt '$config_opt' from '$pool_name': $raw" + exit 1 + fi + + raw=`echo $raw | sed -e 's/[{} "]//g'` + val=`echo $raw | cut -f2 -d:` + + echo "$val" + return 0 +} + +function expect_config_value() +{ + local pool_name config_opt expected_val val + pool_name=$1 + config_opt=$2 + expected_val=$3 + + val=$(get_config_value_or_die $pool_name $config_opt) + + if [[ "$val" != "$expected_val" ]]; then + echo "expected '$expected_val', got '$val'" + exit 1 + fi +} + # note: we need to pass the other args or ceph_argparse.py will take # 'invalid' that is not replicated|erasure and assume it is the next # argument, which is a string. @@ -20,8 +55,11 @@ ceph osd pool create foooo 123 ceph osd pool create foo 123 # idempotent ceph osd pool set foo size 1 +expect_config_value "foo" "min_size" 1 ceph osd pool set foo size 4 +expect_config_value "foo" "min_size" 2 ceph osd pool set foo size 10 +expect_config_value "foo" "min_size" 5 expect_false ceph osd pool set foo size 0 expect_false ceph osd pool set foo size 20 diff --git a/ceph/qa/workunits/rbd/rbd_mirror_ha.sh b/ceph/qa/workunits/rbd/rbd_mirror_ha.sh index 171ab5698..37739a83d 100755 --- a/ceph/qa/workunits/rbd/rbd_mirror_ha.sh +++ b/ceph/qa/workunits/rbd/rbd_mirror_ha.sh @@ -71,7 +71,7 @@ test_replay() wait_for_replay_complete ${CLUSTER1}:${LEADER} ${CLUSTER2} ${POOL} \ ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' \ - 'master_position' \ + 'primary_position' \ "${MIRROR_USER_ID_PREFIX}${LEADER} on $(hostname -s)" if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} \ diff --git a/ceph/qa/workunits/rbd/rbd_mirror_helpers.sh b/ceph/qa/workunits/rbd/rbd_mirror_helpers.sh index 1ec70755b..b5c0a68c5 100755 --- a/ceph/qa/workunits/rbd/rbd_mirror_helpers.sh +++ b/ceph/qa/workunits/rbd/rbd_mirror_helpers.sh @@ -233,6 +233,43 @@ EOF done } +peer_add() +{ + local cluster=$1 ; shift + local pool=$1 ; shift + local client_cluster=$1 ; shift + + local uuid_var_name + if [ -n "$1" ]; then + uuid_var_name=$1 ; shift + fi + + local error_code + local peer_uuid + + for s in 1 2 4 8 16 32; do + set +e + peer_uuid=$(rbd --cluster ${cluster} mirror pool peer add \ + ${pool} ${client_cluster} $@) + error_code=$? + set -e + + if [ $error_code -eq 17 ]; then + # raced with a remote heartbeat ping -- remove and retry + sleep $s + rbd --cluster ${cluster} --pool ${pool} mirror pool peer remove ${peer_uuid} + else + test $error_code -eq 0 + if [ -n "$uuid_var_name" ]; then + eval ${uuid_var_name}=${peer_uuid} + fi + return 0 + fi + done + + return 1 +} + setup_pools() { local cluster=$1 @@ -264,8 +301,8 @@ setup_pools() if [ -z ${RBD_MIRROR_MANUAL_PEERS} ]; then if [ -z ${RBD_MIRROR_CONFIG_KEY} ]; then - rbd --cluster ${cluster} mirror pool peer add ${POOL} ${remote_cluster} - rbd --cluster ${cluster} mirror pool peer add ${PARENT_POOL} ${remote_cluster} + peer_add ${cluster} ${POOL} ${remote_cluster} + peer_add ${cluster} ${PARENT_POOL} ${remote_cluster} else mon_map_file=${TEMPDIR}/${remote_cluster}.monmap CEPH_ARGS='' ceph --cluster ${remote_cluster} mon getmap > ${mon_map_file} @@ -275,12 +312,11 @@ setup_pools() admin_key_file=${TEMPDIR}/${remote_cluster}.client.${CEPH_ID}.key CEPH_ARGS='' ceph --cluster ${remote_cluster} auth get-key client.${CEPH_ID} > ${admin_key_file} - CEPH_ARGS='' rbd --cluster ${cluster} mirror pool peer add ${POOL} \ - client.${CEPH_ID}@${remote_cluster}${PEER_CLUSTER_SUFFIX} \ + CEPH_ARGS='' peer_add ${cluster} ${POOL} \ + client.${CEPH_ID}@${remote_cluster}${PEER_CLUSTER_SUFFIX} '' \ --remote-mon-host "${mon_addr}" --remote-key-file ${admin_key_file} - uuid=$(rbd --cluster ${cluster} mirror pool peer add ${PARENT_POOL} \ - client.${CEPH_ID}@${remote_cluster}${PEER_CLUSTER_SUFFIX}) + peer_add ${cluster} ${PARENT_POOL} client.${CEPH_ID}@${remote_cluster}${PEER_CLUSTER_SUFFIX} uuid CEPH_ARGS='' rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} mon-host ${mon_addr} CEPH_ARGS='' rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} key-file ${admin_key_file} fi diff --git a/ceph/qa/workunits/rbd/rbd_mirror_journal.sh b/ceph/qa/workunits/rbd/rbd_mirror_journal.sh index 426edc42f..da856861b 100755 --- a/ceph/qa/workunits/rbd/rbd_mirror_journal.sh +++ b/ceph/qa/workunits/rbd/rbd_mirror_journal.sh @@ -20,7 +20,7 @@ set_image_meta ${CLUSTER2} ${POOL} ${image} "key2" "value2" wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} write_image ${CLUSTER2} ${POOL} ${image} 100 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown' fi @@ -36,7 +36,7 @@ write_image ${CLUSTER2} ${POOL} ${image1} 100 start_mirrors ${CLUSTER1} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'primary_position' if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown' fi @@ -46,7 +46,7 @@ testlog "TEST: test the first image is replaying after restart" write_image ${CLUSTER2} ${POOL} ${image} 100 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' compare_images ${POOL} ${image} if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then @@ -151,7 +151,7 @@ wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} write_image ${CLUSTER2} ${POOL} ${image} 100 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' compare_images ${POOL} ${image} # failover (unmodified) @@ -170,7 +170,7 @@ wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' promote_image ${CLUSTER2} ${POOL} ${image} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' compare_images ${POOL} ${image} @@ -184,7 +184,7 @@ wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image} write_image ${CLUSTER1} ${POOL} ${image} 100 wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' -wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' 'primary_position' compare_images ${POOL} ${image} # failback @@ -196,7 +196,7 @@ promote_image ${CLUSTER2} ${POOL} ${image} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} write_image ${CLUSTER2} ${POOL} ${image} 100 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' compare_images ${POOL} ${image} @@ -207,7 +207,7 @@ write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' 'primary_position' wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped' promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force' wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${force_promote_image} @@ -233,12 +233,12 @@ write_image ${CLUSTER2} ${POOL} ${clone_image} 100 enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} journal wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' 'primary_position' compare_images ${PARENT_POOL} ${parent_image} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' 'primary_position' compare_images ${POOL} ${clone_image} testlog " - clone v1" @@ -291,7 +291,7 @@ write_image ${CLUSTER2} ${POOL} ${dp_image} 100 create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap2' write_image ${CLUSTER2} ${POOL} ${dp_image} 100 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${dp_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} 'up+replaying' 'primary_position' compare_images ${POOL} ${dp_image}@snap1 compare_images ${POOL} ${dp_image}@snap2 compare_images ${POOL} ${dp_image} @@ -349,17 +349,17 @@ for cluster in ${CLUSTER1} ${CLUSTER2}; do CEPH_ARGS='' rbd --cluster ${cluster} pool init ${pool} rbd --cluster ${cluster} mirror pool enable ${pool} pool done -rbd --cluster ${CLUSTER1} mirror pool peer add ${pool} ${CLUSTER2} -rbd --cluster ${CLUSTER2} mirror pool peer add ${pool} ${CLUSTER1} +peer_add ${CLUSTER1} ${pool} ${CLUSTER2} +peer_add ${CLUSTER2} ${pool} ${CLUSTER1} rdp_image=test_remove_data_pool create_image ${CLUSTER2} ${pool} ${image} 128 create_image ${CLUSTER2} ${POOL} ${rdp_image} 128 --data-pool ${pool} write_image ${CLUSTER2} ${pool} ${image} 100 write_image ${CLUSTER2} ${POOL} ${rdp_image} 100 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${pool} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${pool} ${image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${pool} ${image} 'up+replaying' 'primary_position' wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${rdp_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${rdp_image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${rdp_image} 'up+replaying' 'primary_position' for cluster in ${CLUSTER1} ${CLUSTER2}; do CEPH_ARGS='' ceph --cluster ${cluster} osd pool rm ${pool} ${pool} --yes-i-really-really-mean-it done @@ -405,8 +405,8 @@ write_image ${CLUSTER2} ${POOL}/${NS1} ${image} 100 write_image ${CLUSTER2} ${POOL}/${NS2} ${image} 100 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS2} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${image} 'up+replaying' 'master_position' -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS2} ${image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${image} 'up+replaying' 'primary_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS2} ${image} 'up+replaying' 'primary_position' compare_images ${POOL}/${NS1} ${image} compare_images ${POOL}/${NS2} ${image} @@ -426,7 +426,7 @@ data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL}/${NS1} ${dp_image}) test "${data_pool}" = "${PARENT_POOL}" write_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 100 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${dp_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${dp_image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${dp_image} 'up+replaying' 'primary_position' compare_images ${POOL}/${NS1} ${dp_image} testlog "TEST: simple image resync" @@ -434,7 +434,7 @@ request_resync_image ${CLUSTER1} ${POOL} ${image} image_id wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' compare_images ${POOL} ${image} testlog "TEST: image resync while replayer is stopped" @@ -447,7 +447,7 @@ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' compare_images ${POOL} ${image} fi @@ -458,7 +458,7 @@ start_mirrors ${CLUSTER1} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' compare_images ${POOL} ${image} testlog "TEST: client disconnect" @@ -535,7 +535,7 @@ wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnecte testlog "TEST: split-brain" image=split-brain create_image ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' promote_image ${CLUSTER1} ${POOL} ${image} --force wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' @@ -543,7 +543,7 @@ write_image ${CLUSTER1} ${POOL} ${image} 10 demote_image ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain' request_resync_image ${CLUSTER1} ${POOL} ${image} image_id -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then # teuthology will trash the daemon diff --git a/ceph/qa/workunits/rest/test_mgr_rest_api.py b/ceph/qa/workunits/rest/test_mgr_rest_api.py index 1ed6c757e..f8158af25 100755 --- a/ceph/qa/workunits/rest/test_mgr_rest_api.py +++ b/ceph/qa/workunits/rest/test_mgr_rest_api.py @@ -85,7 +85,7 @@ for method, endpoint, args in screenplay: print("URL = " + url) request = getattr(requests, method)( url, - data=json.dumps(args), + data=json.dumps(args) if args else None, headers=headers, verify=False, auth=auth) diff --git a/ceph/src/.git_version b/ceph/src/.git_version index 03308b0a4..d84310f9b 100644 --- a/ceph/src/.git_version +++ b/ceph/src/.git_version @@ -1,2 +1,2 @@ -9fd2f65f91d9246fae2c841a6222d34d121680ee -15.2.1 +0c857e985a29d90501a285f242ea9c008df49eb8 +15.2.2 diff --git a/ceph/src/CMakeLists.txt b/ceph/src/CMakeLists.txt index 5f370b8db..bb4a7812d 100644 --- a/ceph/src/CMakeLists.txt +++ b/ceph/src/CMakeLists.txt @@ -552,7 +552,7 @@ add_dependencies(ceph-osd erasure_code_plugins) target_link_libraries(ceph-osd osd os global-static common ${BLKID_LIBRARIES}) if(WITH_FUSE) - target_link_libraries(ceph-osd ${FUSE_LIBRARIES}) + target_link_libraries(ceph-osd FUSE::FUSE) endif() set_target_properties(ceph-osd PROPERTIES POSITION_INDEPENDENT_CODE ${EXE_LINKER_USE_PIE} @@ -705,10 +705,9 @@ if(WITH_FUSE) ceph_fuse.cc client/fuse_ll.cc) add_executable(ceph-fuse ${ceph_fuse_srcs}) - target_link_libraries(ceph-fuse ${FUSE_LIBRARIES} + target_link_libraries(ceph-fuse FUSE::FUSE ${GSSAPI_LIBRARIES} client ceph-common global-static ${EXTRALIBS}) set_target_properties(ceph-fuse PROPERTIES - COMPILE_FLAGS "-I${FUSE_INCLUDE_DIRS}" POSITION_INDEPENDENT_CODE ${EXE_LINKER_USE_PIE}) install(TARGETS ceph-fuse DESTINATION bin) install(PROGRAMS mount.fuse.ceph DESTINATION ${CMAKE_INSTALL_SBINDIR}) diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/batch.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/batch.py index 27ee5bb47..b7a4b35b2 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/batch.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/batch.py @@ -106,9 +106,12 @@ def filter_devices(args): if len(unused_devices) == 1: last_device = unused_devices[0] if not last_device.rotational and last_device.is_lvm_member: - reason = "Used by ceph as a %s already and there are no devices left for data/block" % ( - last_device.lvs[0].tags.get("ceph.type"), - ) + if last_device.lvs: + reason = "Used by ceph as a %s already and there are no devices left for data/block" % ( + last_device.lvs[0].tags.get("ceph.type"), + ) + else: + reason = "Disk is an LVM member already, skipping" filtered_devices[last_device.abspath] = {"reasons": [reason]} logger.info(reason + ": %s" % last_device.abspath) unused_devices = [] @@ -366,7 +369,12 @@ class Batch(object): self.filtered_devices.update({d: used_reason for d in getattr(self.args, dev_list_prop) if d.used_by_ceph}) - if self.args.yes and dev_list and devs != usable: + # only fail if non-interactive, this iteration concerns + # non-data devices, there are usable data devices (or not all + # data devices were filtered) and non-data devices were filtered + # so in short this branch is not taken if all data devices are + # filtered + if self.args.yes and dev_list and self.usable and devs != usable: err = '{} devices were filtered in non-interactive mode, bailing out' raise RuntimeError(err.format(len(devs) - len(usable))) diff --git a/ceph/src/ceph-volume/ceph_volume/devices/simple/scan.py b/ceph/src/ceph-volume/ceph_volume/devices/simple/scan.py index 6e41f76ec..1e3deae4c 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/simple/scan.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/simple/scan.py @@ -103,8 +103,9 @@ class Scan(object): file_json_key = file_ if file_.endswith('_dmcrypt'): file_json_key = file_.rstrip('_dmcrypt') - logger.info(('reading file {}, stripping _dmcrypt', - 'suffix').format(file_)) + logger.info( + 'reading file {}, stripping _dmcrypt suffix'.format(file_) + ) if os.path.islink(file_path): if os.path.exists(file_path): osd_metadata[file_json_key] = self.scan_device(file_path) diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/test_explicit.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/test_explicit.yml index ac01062a0..f3be4972a 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/test_explicit.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/test_explicit.yml @@ -9,7 +9,6 @@ state: stopped with_items: "{{ osd_ids }}" - - hosts: mons become: yes tasks: @@ -21,7 +20,6 @@ command: "ceph --cluster {{ cluster }} osd purge osd.{{ item }} --yes-i-really-mean-it" with_items: "{{ osd_ids }}" - - hosts: osds become: yes tasks: @@ -37,7 +35,7 @@ environment: CEPH_VOLUME_DEBUG: 1 - - name: ensure batch create is idempotent + - name: ensure batch create is idempotent when all data devices are filtered command: "ceph-volume --cluster {{ cluster }} lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices[:2] | join(' ') }} --db-devices {{ devices[2:] | join(' ') }}" register: batch_cmd failed_when: false @@ -49,7 +47,6 @@ msg: "lvm batch failed idempotency check" when: - batch_cmd.rc != 0 - - "'strategy changed' not in batch_cmd.stderr" - name: run batch --report to see if devices get filtered command: "ceph-volume --cluster {{ cluster }} lvm batch --report --format=json --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices[:2] | join(' ') }} --db-devices {{ devices[2:] | join(' ') }}" @@ -63,4 +60,3 @@ msg: "lvm batch --report failed idempotency check" when: - report_cmd.rc != 0 - - "'strategy changed' not in report_cmd.stderr" diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/playbooks/deploy.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/playbooks/deploy.yml index 12bca4f32..0c1d13f8f 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/playbooks/deploy.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/playbooks/deploy.yml @@ -90,6 +90,8 @@ tasks: - import_role: name: ceph-defaults + - import_role: + name: ceph-facts - import_role: name: ceph-validate diff --git a/ceph/src/ceph.in b/ceph/src/ceph.in index aedec606f..369b96965 100755 --- a/ceph/src/ceph.in +++ b/ceph/src/ceph.in @@ -333,7 +333,7 @@ def parse_cmdargs(args=None, target=''): help="make less verbose") parser.add_argument('-f', '--format', choices=['json', 'json-pretty', - 'xml', 'xml-pretty', 'plain'], dest='output_format') + 'xml', 'xml-pretty', 'plain', 'yaml'], dest='output_format') parser.add_argument('--connect-timeout', dest='cluster_timeout', type=int, diff --git a/ceph/src/ceph_fuse.cc b/ceph/src/ceph_fuse.cc index 5ac18116f..9d6141c1e 100644 --- a/ceph/src/ceph_fuse.cc +++ b/ceph/src/ceph_fuse.cc @@ -41,6 +41,7 @@ #include #include +#include #define dout_context g_ceph_context @@ -51,7 +52,12 @@ static void fuse_usage() "-h", }; struct fuse_args args = FUSE_ARGS_INIT(2, (char**)argv); +#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0) + struct fuse_cmdline_opts opts = {}; + if (fuse_parse_cmdline(&args, &opts) == -1) { +#else if (fuse_parse_cmdline(&args, nullptr, nullptr, nullptr) == -1) { +#endif derr << "fuse_parse_cmdline failed." << dendl; } ceph_assert(args.allocated); @@ -105,7 +111,12 @@ int main(int argc, const char **argv, const char *envp[]) { }; struct fuse_args fargs = FUSE_ARGS_INIT(2, (char**)tmpargv); +#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0) + struct fuse_cmdline_opts opts = {}; + if (fuse_parse_cmdline(&fargs, &opts) == -1) { +#else if (fuse_parse_cmdline(&fargs, nullptr, nullptr, nullptr) == -1) { +#endif derr << "fuse_parse_cmdline failed." << dendl; } ceph_assert(fargs.allocated); diff --git a/ceph/src/cephadm/cephadm b/ceph/src/cephadm/cephadm index baccd7c05..32610e702 100755 --- a/ceph/src/cephadm/cephadm +++ b/ceph/src/cephadm/cephadm @@ -1,6 +1,8 @@ #!/usr/bin/python3 DEFAULT_IMAGE='docker.io/ceph/ceph:v15' +DEFAULT_IMAGE_IS_MASTER=False +LATEST_STABLE_RELEASE='octopus' DATA_DIR='/var/lib/ceph' LOG_DIR='/var/log/ceph' LOCK_DIR='/run/cephadm' @@ -190,6 +192,8 @@ class NFSGanesha(object): # config-json options self.pool = json_get('pool', require=True) self.namespace = json_get('namespace') + self.userid = json_get('userid') + self.extra_args = json_get('extra_args', []) self.files = json_get('files', {}) # validate the supplied args @@ -264,6 +268,10 @@ class NFSGanesha(object): cname = '%s-%s' % (cname, desc) return cname + def get_daemon_args(self): + # type: () -> List[str] + return self.daemon_args + self.extra_args + def get_file_content(self, fname): # type: (str) -> str """Normalize the json file content into a string""" @@ -303,6 +311,8 @@ class NFSGanesha(object): args=['--pool', self.pool] if self.namespace: args += ['--ns', self.namespace] + if self.userid: + args += ['--userid', self.userid] args += [action, self.get_daemon_name()] data_dir = get_data_dir(self.fsid, self.daemon_type, self.daemon_id) @@ -322,10 +332,135 @@ class NFSGanesha(object): ################################## +class CephIscsi(object): + """Defines a Ceph-Iscsi container""" + + daemon_type = 'iscsi' + entrypoint = '/usr/bin/rbd-target-api' + + required_files = ['iscsi-gateway.cfg'] + + def __init__(self, + fsid, + daemon_id, + config_json, + image=DEFAULT_IMAGE): + # type: (str, Union[int, str], Dict, str) -> None + self.fsid = fsid + self.daemon_id = daemon_id + self.image = image + + def json_get(key, default=None, require=False): + if require and not key in config_json.keys(): + raise Error('{} missing from config-json'.format(key)) + return config_json.get(key, default) + + # config-json options + self.files = json_get('files', {}) + + # validate the supplied args + self.validate() + + @classmethod + def init(cls, fsid, daemon_id): + # type: (str, Union[int, str]) -> CephIscsi + return cls(fsid, daemon_id, get_parm(args.config_json), args.image) + + @staticmethod + def get_container_mounts(data_dir, log_dir): + # type: (str, str) -> Dict[str, str] + mounts = dict() + mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z' + mounts[os.path.join(data_dir, 'keyring')] = '/etc/ceph/keyring:z' + mounts[os.path.join(data_dir, 'iscsi-gateway.cfg')] = '/etc/ceph/iscsi-gateway.cfg:z' + mounts[os.path.join(data_dir, 'configfs')] = '/sys/kernel/config:z' + mounts[log_dir] = '/var/log/rbd-target-api:z' + mounts['/dev/log'] = '/dev/log:z' + return mounts + + @staticmethod + def get_version(container_id): + # type(str) -> Optional[str] + version = None + out, err, code = call( + [container_path, 'exec', container_id, + '/usr/bin/python3', '-c', "import pkg_resources; print(pkg_resources.require('ceph_iscsi')[0].version)"]) + if code == 0: + version = out + return version + + def validate(self): + # type () -> None + if not is_fsid(self.fsid): + raise Error('not an fsid: %s' % self.fsid) + if not self.daemon_id: + raise Error('invalid daemon_id: %s' % self.daemon_id) + if not self.image: + raise Error('invalid image: %s' % self.image) + + # check for the required files + if self.required_files: + for fname in self.required_files: + if fname not in self.files: + raise Error('required file missing from config-json: %s' % fname) + + def get_daemon_name(self): + # type: () -> str + return '%s.%s' % (self.daemon_type, self.daemon_id) + + def get_container_name(self, desc=None): + # type: (Optional[str]) -> str + cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name()) + if desc: + cname = '%s-%s' % (cname, desc) + return cname + + def get_file_content(self, fname): + # type: (str) -> str + """Normalize the json file content into a string""" + content = self.files.get(fname) + if isinstance(content, list): + content = '\n'.join(content) + return content + + def create_daemon_dirs(self, data_dir, uid, gid): + # type: (str, int, int) -> None + """Create files under the container data dir""" + if not os.path.isdir(data_dir): + raise OSError('data_dir is not a directory: %s' % (data_dir)) + + logger.info('Creating ceph-iscsi config...') + configfs_dir = os.path.join(data_dir, 'configfs') + makedirs(configfs_dir, uid, gid, 0o755) + + # populate files from the config-json + for fname in self.files: + config_file = os.path.join(data_dir, fname) + config_content = self.get_file_content(fname) + logger.info('Write file: %s' % (config_file)) + with open(config_file, 'w') as f: + os.fchown(f.fileno(), uid, gid) + os.fchmod(f.fileno(), 0o600) + f.write(config_content) + + @staticmethod + def configfs_mount_umount(data_dir, mount=True): + mount_path = os.path.join(data_dir, 'configfs') + if mount: + cmd = "if ! grep -qs {0} /proc/mounts; then " \ + "mount -t configfs none {0}; fi".format(mount_path) + else: + cmd = "if grep -qs {0} /proc/mounts; then " \ + "umount {0}; fi".format(mount_path) + return cmd.split() + +################################## + def get_supported_daemons(): supported_daemons = list(Ceph.daemons) supported_daemons.extend(Monitoring.components) supported_daemons.append(NFSGanesha.daemon_type) + supported_daemons.append(CephIscsi.daemon_type) assert len(supported_daemons) == len(set(supported_daemons)) return supported_daemons @@ -366,6 +501,8 @@ def check_ip_port(ip, port): logger.info('Verifying IP %s port %d ...' % (ip, port)) if ip.startswith('[') or '::' in ip: s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + if ip.startswith('[') and ip.endswith(']'): + ip = ip[1:-1] else: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: @@ -941,6 +1078,18 @@ def infer_fsid(func): return _infer_fsid +def _get_default_image(): + if DEFAULT_IMAGE_IS_MASTER: + yellow = '\033[93m' + end = '\033[0m' + warn = '''This is a development version of cephadm. +For information regarding the latest stable release: + https://docs.ceph.com/docs/{}/cephadm/install +'''.format(LATEST_STABLE_RELEASE) + for line in warn.splitlines(): + logger.warning('{}{}{}'.format(yellow, line, end)) + return DEFAULT_IMAGE + def infer_image(func): """ Use the most recent ceph image @@ -952,7 +1101,7 @@ def infer_image(func): if not args.image: args.image = get_last_local_ceph_image() if not args.image: - args.image = DEFAULT_IMAGE + args.image = _get_default_image() return func() return _infer_image @@ -968,7 +1117,8 @@ def default_image(func): if not args.image: args.image = os.environ.get('CEPHADM_IMAGE') if not args.image: - args.image = DEFAULT_IMAGE + args.image = _get_default_image() + return func() return _default_image @@ -1274,13 +1424,13 @@ def get_daemon_args(fsid, daemon_type, daemon_id): for peer in peers: r += ["--cluster.peer={}".format(peer)] elif daemon_type == NFSGanesha.daemon_type: - r += NFSGanesha.daemon_args + nfs_ganesha = NFSGanesha.init(fsid, daemon_id) + r += nfs_ganesha.get_daemon_args() return r def create_daemon_dirs(fsid, daemon_type, daemon_id, uid, gid, - config=None, keyring=None, - reconfig=False): + config=None, keyring=None, reconfig=False): # type: (str, str, Union[int, str], int, int, Optional[str], Optional[str], Optional[bool]) -> None data_dir = make_data_dir(fsid, daemon_type, daemon_id, uid=uid, gid=gid) make_log_dir(fsid, uid=uid, gid=gid) @@ -1341,6 +1491,10 @@ def create_daemon_dirs(fsid, daemon_type, daemon_id, uid, gid, nfs_ganesha = NFSGanesha.init(fsid, daemon_id) nfs_ganesha.create_daemon_dirs(data_dir, uid, gid) + if daemon_type == CephIscsi.daemon_type: + ceph_iscsi = CephIscsi.init(fsid, daemon_id) + ceph_iscsi.create_daemon_dirs(data_dir, uid, gid) + def get_parm(option): # type: (str) -> Dict[str, str] @@ -1456,6 +1610,12 @@ def get_container_mounts(fsid, daemon_type, daemon_id, data_dir = get_data_dir(fsid, daemon_type, daemon_id) mounts.update(NFSGanesha.get_container_mounts(data_dir)) + if daemon_type == CephIscsi.daemon_type: + assert daemon_id + data_dir = get_data_dir(fsid, daemon_type, daemon_id) + log_dir = get_log_dir(fsid) + mounts.update(CephIscsi.get_container_mounts(data_dir, log_dir)) + return mounts def get_container(fsid, daemon_type, daemon_id, @@ -1484,6 +1644,9 @@ def get_container(fsid, daemon_type, daemon_id, elif daemon_type == NFSGanesha.daemon_type: entrypoint = NFSGanesha.entrypoint name = '%s.%s' % (daemon_type, daemon_id) + elif daemon_type == CephIscsi.daemon_type: + entrypoint = CephIscsi.entrypoint + name = '%s.%s' % (daemon_type, daemon_id) else: entrypoint = '' name = '' @@ -1643,6 +1806,12 @@ def deploy_daemon_units(fsid, uid, gid, daemon_type, daemon_id, c, nfs_ganesha = NFSGanesha.init(fsid, daemon_id) prestart = nfs_ganesha.get_rados_grace_container('add') f.write(' '.join(prestart.run_cmd()) + '\n') + elif daemon_type == CephIscsi.daemon_type: + f.write(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=True)) + '\n') + + if daemon_type in Ceph.daemons: + install_path = find_program('install') + f.write('{install_path} -d -m0770 -o {uid} -g {gid} /var/run/ceph/{fsid}\n'.format(install_path=install_path, fsid=fsid, uid=uid, gid=gid)) # container run command f.write(' '.join(c.run_cmd()) + '\n') @@ -1672,6 +1841,8 @@ def deploy_daemon_units(fsid, uid, gid, daemon_type, daemon_id, c, nfs_ganesha = NFSGanesha.init(fsid, daemon_id) poststop = nfs_ganesha.get_rados_grace_container('remove') f.write(' '.join(poststop.run_cmd()) + '\n') + elif daemon_type == CephIscsi.daemon_type: + f.write(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=False)) + '\n') os.fchmod(f.fileno(), 0o600) os.rename(data_dir + '/unit.poststop.new', data_dir + '/unit.poststop') @@ -1684,7 +1855,7 @@ def deploy_daemon_units(fsid, uid, gid, daemon_type, daemon_id, c, # systemd install_base_units(fsid) - unit = get_unit_file(fsid, uid, gid) + unit = get_unit_file(fsid) unit_file = 'ceph-%s@.service' % (fsid) with open(args.unit_dir + '/' + unit_file + '.new', 'w') as f: f.write(unit) @@ -1821,9 +1992,8 @@ def install_base_units(fsid): } """ % fsid) -def get_unit_file(fsid, uid, gid): - # type: (str, int, int) -> str - install_path = find_program('install') +def get_unit_file(fsid): + # type: (str) -> str u = """# generated by cephadm [Unit] Description=Ceph %i for {fsid} @@ -1843,7 +2013,6 @@ LimitNOFILE=1048576 LimitNPROC=1048576 EnvironmentFile=-/etc/environment ExecStartPre=-{container_path} rm ceph-{fsid}-%i -ExecStartPre=-{install_path} -d -m0770 -o {uid} -g {gid} /var/run/ceph/{fsid} ExecStart=/bin/bash {data_dir}/{fsid}/%i/unit.run ExecStop=-{container_path} stop ceph-{fsid}-%i ExecStopPost=-/bin/bash {data_dir}/{fsid}/%i/unit.poststop @@ -1859,10 +2028,7 @@ StartLimitBurst=5 WantedBy=ceph-{fsid}.target """.format( container_path=container_path, - install_path=install_path, fsid=fsid, - uid=uid, - gid=gid, data_dir=args.data_dir) return u @@ -2415,7 +2581,7 @@ def command_bootstrap(): cmd = ['dashboard', 'ac-user-create', args.initial_dashboard_user, password, 'administrator', '--force-password'] if not args.dashboard_password_noupdate: cmd.append('--pwd-update-required') - cli(cmd) + cli(cmd) logger.info('Fetching dashboard port number...') out = cli(['config', 'get', 'mgr', 'mgr/dashboard/ssl_server_port']) port = int(out) @@ -2521,6 +2687,13 @@ def command_deploy(): deploy_daemon(args.fsid, daemon_type, daemon_id, c, uid, gid, config=config, keyring=keyring, reconfig=args.reconfig) + elif daemon_type == CephIscsi.daemon_type: + (config, keyring) = get_config_and_keyring() + (uid, gid) = extract_uid_gid() + c = get_container(args.fsid, daemon_type, daemon_id) + deploy_daemon(args.fsid, daemon_type, daemon_id, c, uid, gid, + config=config, keyring=keyring, + reconfig=args.reconfig) else: raise Error("{} not implemented in command_deploy function".format(daemon_type)) @@ -2621,8 +2794,12 @@ def command_enter(): '-e', 'LANG=C', '-e', "PS1=%s" % CUSTOM_PS1, ] - c = get_container(args.fsid, daemon_type, daemon_id, - container_args=container_args) + c = CephContainer( + image=args.image, + entrypoint='doesnotmatter', + container_args=container_args, + cname='ceph-%s-%s.%s' % (args.fsid, daemon_type, daemon_id), + ) command = c.exec_cmd(command) return call_timeout(command, args.timeout) @@ -2635,6 +2812,9 @@ def command_ceph_volume(): if args.fsid: make_log_dir(args.fsid) + l = FileLock(args.fsid) + l.acquire() + (uid, gid) = (0, 0) # ceph-volume runs as root mounts = get_container_mounts(args.fsid, 'osd', None) @@ -2828,6 +3008,8 @@ def list_daemons(detail=True, legacy_dir=None): version = seen_versions.get(image_id, None) if daemon_type == NFSGanesha.daemon_type: version = NFSGanesha.get_version(container_id) + if daemon_type == CephIscsi.daemon_type: + version = CephIscsi.get_version(container_id) elif not version: if daemon_type in Ceph.daemons: out, err, code = call( @@ -2926,6 +3108,84 @@ def command_adopt(): raise Error('daemon type %s not recognized' % daemon_type) +class AdoptOsd(object): + def __init__(self, osd_data_dir, osd_id): + # type: (str, str) -> None + self.osd_data_dir = osd_data_dir + self.osd_id = osd_id + + def check_online_osd(self): + # type: () -> Tuple[Optional[str], Optional[str]] + + osd_fsid, osd_type = None, None + + path = os.path.join(self.osd_data_dir, 'fsid') + try: + with open(path, 'r') as f: + osd_fsid = f.read().strip() + logger.info("Found online OSD at %s" % path) + if os.path.exists(os.path.join(self.osd_data_dir, 'type')): + with open(os.path.join(self.osd_data_dir, 'type')) as f: + osd_type = f.read().strip() + else: + logger.info('"type" file missing for OSD data dir') + except IOError: + logger.info('Unable to read OSD fsid from %s' % path) + + return osd_fsid, osd_type + + def check_offline_lvm_osd(self): + # type: () -> Tuple[Optional[str], Optional[str]] + + osd_fsid, osd_type = None, None + + c = CephContainer( + image=args.image, + entrypoint='/usr/sbin/ceph-volume', + args=['lvm', 'list', '--format=json'], + privileged=True + ) + out, err, code = call_throws(c.run_cmd(), verbose=False) + if not code: + try: + js = json.loads(out) + if self.osd_id in js: + logger.info("Found offline LVM OSD {}".format(self.osd_id)) + osd_fsid = js[self.osd_id][0]['tags']['ceph.osd_fsid'] + for device in js[self.osd_id]: + if device['tags']['ceph.type'] == 'block': + osd_type = 'bluestore' + break + if device['tags']['ceph.type'] == 'data': + osd_type = 'filestore' + break + except ValueError as e: + logger.info("Invalid JSON in ceph-volume lvm list: {}".format(e)) + + return osd_fsid, osd_type + + def check_offline_simple_osd(self): + # type: () -> Tuple[Optional[str], Optional[str]] + + osd_fsid, osd_type = None, None + + osd_file = glob("/etc/ceph/osd/{}-[a-f0-9-]*.json".format(self.osd_id)) + if len(osd_file) == 1: + with open(osd_file[0], 'r') as f: + try: + js = json.loads(f.read()) + logger.info("Found offline simple OSD {}".format(self.osd_id)) + osd_fsid = js["fsid"] + osd_type = js["type"] + if osd_type != "filestore": + # need this to be mounted for the adopt to work, as it + # needs to move files from this directory + call_throws(['mount', js["data"]["path"], self.osd_data_dir]) + except ValueError as e: + logger.info("Invalid JSON in {}: {}".format(osd_file, e)) + + return osd_fsid, osd_type + def command_adopt_ceph(daemon_type, daemon_id, fsid): # type: (str, str, str) -> None @@ -2936,22 +3196,23 @@ def command_adopt_ceph(daemon_type, daemon_id, fsid): (daemon_type, args.cluster, daemon_id)) data_dir_src = os.path.abspath(args.legacy_dir + data_dir_src) + if not os.path.exists(data_dir_src): + raise Error("{}.{} data directory '{}' does not exist. " + "Incorrect ID specified, or daemon alrady adopted?".format( + daemon_type, daemon_id, data_dir_src)) + osd_fsid = None if daemon_type == 'osd': - path = os.path.join(data_dir_src, 'fsid') - try: - with open(path, 'r') as f: - osd_fsid = f.read().strip() - except IOError: - raise Error('unable to read OSD fsid from %s' % path) - os_type = None - if os.path.exists(os.path.join(data_dir_src, 'type')): - with open(os.path.join(data_dir_src, 'type')) as f: - os_type = f.read().strip() - else: - raise Error('"type" file missing for OSD data dir') - logger.info('objectstore_type is %s' % os_type) - if os_type == 'filestore': + adopt_osd = AdoptOsd(data_dir_src, daemon_id) + osd_fsid, osd_type = adopt_osd.check_online_osd() + if not osd_fsid: + osd_fsid, osd_type = adopt_osd.check_offline_lvm_osd() + if not osd_fsid: + osd_fsid, osd_type = adopt_osd.check_offline_simple_osd() + if not osd_fsid: + raise Error('Unable to find OSD {}'.format(daemon_id)) + logger.info('objectstore_type is %s' % osd_type) + if osd_type == 'filestore': raise Error('FileStore is not supported by cephadm') # NOTE: implicit assumption here that the units correspond to the @@ -3010,9 +3271,8 @@ def command_adopt_ceph(daemon_type, daemon_id, fsid): logger.info('Renaming %s -> %s', simple_fn, new_fn) os.rename(simple_fn, new_fn) logger.info('Disabling host unit ceph-volume@ simple unit...') - call_throws(['systemctl', 'disable', - 'ceph-volume@simple-%s-%s.service' % ( - daemon_id, osd_fsid)]) + call(['systemctl', 'disable', + 'ceph-volume@simple-%s-%s.service' % (daemon_id, osd_fsid)]) else: # assume this is an 'lvm' c-v for now, but don't error # out if it's not. @@ -3041,7 +3301,7 @@ def command_adopt_ceph(daemon_type, daemon_id, fsid): c = get_container(fsid, daemon_type, daemon_id) deploy_daemon_units(fsid, uid, gid, daemon_type, daemon_id, c, enable=True, # unconditionally enable the new unit - start=(state == 'running'), + start=(state == 'running' or args.force_start), osd_fsid=osd_fsid) update_firewalld(daemon_type) @@ -3061,6 +3321,7 @@ def command_adopt_prometheus(daemon_id, fsid): config_src = '/etc/prometheus/prometheus.yml' config_src = os.path.abspath(args.legacy_dir + config_src) config_dst = os.path.join(data_dir_dst, 'etc/prometheus') + makedirs(config_dst, uid, gid, 0o755) copy_files([config_src], config_dst, uid=uid, gid=gid) # data @@ -3283,6 +3544,19 @@ def command_rm_cluster(): # rm logrotate config call_throws(['rm', '-f', args.logrotate_dir + '/ceph-%s' % args.fsid]) + # clean up config, keyring, and pub key files + files = ['/etc/ceph/ceph.conf', '/etc/ceph/ceph.pub', '/etc/ceph/ceph.client.admin.keyring'] + + if os.path.exists(files[0]): + valid_fsid = False + with open(files[0]) as f: + if args.fsid in f.read(): + valid_fsid = True + if valid_fsid: + for n in range(0, len(files)): + if os.path.exists(files[n]): + os.remove(files[n]) + ################################## @@ -3302,29 +3576,44 @@ def check_time_sync(enabler=None): def command_check_host(): # type: () -> None - # caller already checked for docker/podman - logger.info('podman|docker (%s) is present' % container_path) - + errors = [] commands = ['systemctl', 'lvcreate'] + if args.docker: + container_path = find_program('docker') + else: + for i in CONTAINER_PREFERENCE: + try: + container_path = find_program(i) + break + except Exception as e: + logger.debug('Could not locate %s: %s' % (i, e)) + if not container_path: + errors.append('Unable to locate any of %s' % CONTAINER_PREFERENCE) + else: + logger.info('podman|docker (%s) is present' % container_path) + for command in commands: try: find_program(command) logger.info('%s is present' % command) except ValueError: - raise Error('%s binary does not appear to be installed' % command) + errors.append('%s binary does not appear to be installed' % command) # check for configured+running chronyd or ntp if not check_time_sync(): - raise Error('No time synchronization is active') + errors.append('No time synchronization is active') if 'expect_hostname' in args and args.expect_hostname: - if get_hostname() != args.expect_hostname: - raise Error('hostname "%s" does not match expected hostname "%s"' % ( + if get_hostname().lower() != args.expect_hostname.lower(): + errors.append('hostname "%s" does not match expected hostname "%s"' % ( get_hostname(), args.expect_hostname)) logger.info('Hostname "%s" matches what is expected.', args.expect_hostname) + if errors: + raise Error('\n'.join(errors)) + logger.info('Host looks OK') ################################## @@ -3784,6 +4073,8 @@ def create_packager(stable=None, version=None, branch=None, commit=None): def command_add_repo(): if args.version and args.release: raise Error('you can specify either --release or --version but not both') + if not args.version and not args.release and not args.dev and not args.dev_commit: + raise Error('please supply a --release, --version, --dev or --dev-commit argument') if args.version: try: (x, y, z) = args.version.split('.') @@ -3910,6 +4201,10 @@ def _get_parser(): '--skip-pull', action='store_true', help='do not pull the latest image before adopting') + parser_adopt.add_argument( + '--force-start', + action='store_true', + help="start newly adoped daemon, even if it wasn't running previously") parser_rm_daemon = subparsers.add_parser( 'rm-daemon', help='remove daemon instance') @@ -4204,7 +4499,7 @@ def _get_parser(): parser_add_repo.set_defaults(func=command_add_repo) parser_add_repo.add_argument( '--release', - help='use latest version of a named release (e.g., octopus)') + help='use latest version of a named release (e.g., {})'.format(LATEST_STABLE_RELEASE)) parser_add_repo.add_argument( '--version', help='use specific upstream version (x.y.z)') @@ -4260,24 +4555,26 @@ if __name__ == "__main__": sys.stderr.write('ERROR: cephadm should be run as root\n') sys.exit(1) - # podman or docker? - if args.docker: - container_path = find_program('docker') - else: - for i in CONTAINER_PREFERENCE: - try: - container_path = find_program(i) - break - except Exception as e: - logger.debug('Could not locate %s: %s' % (i, e)) - if not container_path and args.func != command_prepare_host: - sys.stderr.write('Unable to locate any of %s\n' % CONTAINER_PREFERENCE) - sys.exit(1) - if 'func' not in args: sys.stderr.write('No command specified; pass -h or --help for usage\n') sys.exit(1) + # podman or docker? + if args.func != command_check_host: + if args.docker: + container_path = find_program('docker') + else: + for i in CONTAINER_PREFERENCE: + try: + container_path = find_program(i) + break + except Exception as e: + logger.debug('Could not locate %s: %s' % (i, e)) + if not container_path and args.func != command_prepare_host\ + and args.func != command_add_repo: + sys.stderr.write('Unable to locate any of %s\n' % CONTAINER_PREFERENCE) + sys.exit(1) + try: r = args.func() except Error as e: diff --git a/ceph/src/cephadm/samples/rgw_ssl.json b/ceph/src/cephadm/samples/rgw_ssl.json new file mode 100644 index 000000000..d3c45111a --- /dev/null +++ b/ceph/src/cephadm/samples/rgw_ssl.json @@ -0,0 +1,103 @@ +{ + "rgw_realm": "default", + "rgw_zone": "default", + "service_type": "rgw", + "placement": { + "hosts": [{ + "hostname": "ironic-moliver", + "name": "", + "network": "" + }], + "count": 1 + }, + "ssl": true, + "rgw_frontend_port": 4343, + "rgw_frontend_ssl_certificate": [ + "-----BEGIN CERTIFICATE-----", + "MIIFmjCCA4KgAwIBAgIJAIZ2n35bmwXTMA0GCSqGSIb3DQEBCwUAMGIxCzAJBgNV", + "BAYTAkFVMQwwCgYDVQQIDANOU1cxHTAbBgNVBAoMFEV4YW1wbGUgUkdXIFNTTCBp", + "bmMuMSYwJAYDVQQDDB1yZ3ctZW5kcG9pbnQuZXhhbXBsZS1jZXBoLmNvbTAeFw0y", + "MDAyMDcwMDEzNTFaFw0zMDAyMDQwMDEzNTFaMGIxCzAJBgNVBAYTAkFVMQwwCgYD", + "VQQIDANOU1cxHTAbBgNVBAoMFEV4YW1wbGUgUkdXIFNTTCBpbmMuMSYwJAYDVQQD", + "DB1yZ3ctZW5kcG9pbnQuZXhhbXBsZS1jZXBoLmNvbTCCAiIwDQYJKoZIhvcNAQEB", + "BQADggIPADCCAgoCggIBAMptGJ523QkEbc37za8iuCTahj0Zr6hy+ToSX/Vfdzxj", + "iYHuD2PiZZyJB7t2eOqiA8sQ5N513EUtf2ZIBwtnnqFIzD5TqI3BxRajUTlOyXUX", + "onMwQwXu2ifDUy3LCmuQfzanOTWvVLac1NmkWbJHpJCXYbUnPb1Nvd0QjTTEH1jt", + "5bDHhfxwCIYK6PY+MqC72a09wB2ZF+EKsSdqghOKmibfJHtoJdsqGeLrysBLrzUJ", + "e/5ZW3V4Z85T2lja5KZnWgRofrUy5TmJV10HO4Hht92xvWvEi/rmjg2AVYZFUQQx", + "xKXpUBbF5T46eSVmaT7IH88Yp5ytgBTaigym7ETCjohp/DfCaK1DUehh0ce7iUq2", + "yCLviZsX4WdPYxzkoLflNrqm4YZP6iKcZSUR/A+qPKCzCXgMXFNA1JxilDwEq35F", + "zGN++ehJqdNmOQ1eQScsLwZQa6mC97d+upWdCvyntf1+S6vNcXhtRQpjNM4W37oW", + "r5nicsGA3/0rpDEHZW85KlkdWO1uCS/6ftgt8UUMaf5ew3PigzusqymBWTlMOjtW", + "uAQXxgZZvkRp+xdspn/uTCAP+bNShGD6Q+TO3U6IjTqHk83sGKCvg2dyU/dqgPr9", + "2IIzgQBFGk0W0nM/E83E8hUSwX17COLL3drhPZb4VRMChQ8PAa6u9nIymkX2wSVv", + "AgMBAAGjUzBRMB0GA1UdDgQWBBSsZHuY7KK80RrZHp+Gx+k16skuRDAfBgNVHSME", + "GDAWgBSsZHuY7KK80RrZHp+Gx+k16skuRDAPBgNVHRMBAf8EBTADAQH/MA0GCSqG", + "SIb3DQEBCwUAA4ICAQAE+BLtnu0p8FtK7vrBCRcCdvycWaSFGJUt7r5Nm8TD7sKw", + "bWeDLgXrRouyA7n6yt/JqQbXYcxt4MLAM0P6NQd5BlNrrnDk4rBnJiJgejppNE+S", + "BazR7Dv0uYcs8kPT4DPpwzv4aJ2aXCBaxYrq8Rx2xOqANCPVOrtPUk9yGpaQ5adU", + "GfxkVbpgIEz1c71PeQuK1KUU/Wpk7cpm+FQCizl9ftP2lHWsGhSLCuyWoMTjt68P", + "gYEWoV54eo/bzwj2ei6TcfNo+uHyzEiiG2qEvMh/cnYUFzs8O1t0mN19WPB1pSh1", + "faci5lGdtkRbLgP0g5RvpagE7Lw3mCc5Om8jmHs4mPfuVkssBVV23CrFpqLLrDX3", + "Acwb/zRGvA7T4WESBTJMYFOLgm0W0Y+AN8RcYNU9QbDhe++Te0uz/3Sy3GN2Xg5z", + "MxfD1+34x6KvMfCh8NjII2mFQ9ukcfrhcfO3oWDLlwsqlVbhkZxNiUOEIx9nzHcF", + "kWpZ2ypBDH45h2o3LyqvGjsu/BFkeG6JpEDCWbClKWcjKxOrLVDufhSDduffDjja", + "zOsgQJg0Yf//Ubb5p0c54GjHM/XDXEcV3m3sEtbmMYz6xGwuag4bx8P2E/QY8sFp", + "JxgIdS8vdl6YhDCjKJ2XzI30JwCdftgDIAiWSE0ivoDc+8+gG1nb11GT52HFzA==", + "-----END CERTIFICATE-----" + ], + "rgw_frontend_ssl_key": [ + "-----BEGIN PRIVATE KEY-----", + "MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDKbRiedt0JBG3N", + "+82vIrgk2oY9Ga+ocvk6El/1X3c8Y4mB7g9j4mWciQe7dnjqogPLEOTeddxFLX9m", + "SAcLZ56hSMw+U6iNwcUWo1E5Tsl1F6JzMEMF7tonw1MtywprkH82pzk1r1S2nNTZ", + "pFmyR6SQl2G1Jz29Tb3dEI00xB9Y7eWwx4X8cAiGCuj2PjKgu9mtPcAdmRfhCrEn", + "aoITipom3yR7aCXbKhni68rAS681CXv+WVt1eGfOU9pY2uSmZ1oEaH61MuU5iVdd", + "BzuB4bfdsb1rxIv65o4NgFWGRVEEMcSl6VAWxeU+OnklZmk+yB/PGKecrYAU2ooM", + "puxEwo6Iafw3wmitQ1HoYdHHu4lKtsgi74mbF+FnT2Mc5KC35Ta6puGGT+oinGUl", + "EfwPqjygswl4DFxTQNScYpQ8BKt+RcxjfvnoSanTZjkNXkEnLC8GUGupgve3frqV", + "nQr8p7X9fkurzXF4bUUKYzTOFt+6Fq+Z4nLBgN/9K6QxB2VvOSpZHVjtbgkv+n7Y", + "LfFFDGn+XsNz4oM7rKspgVk5TDo7VrgEF8YGWb5EafsXbKZ/7kwgD/mzUoRg+kPk", + "zt1OiI06h5PN7Bigr4NnclP3aoD6/diCM4EARRpNFtJzPxPNxPIVEsF9ewjiy93a", + "4T2W+FUTAoUPDwGurvZyMppF9sElbwIDAQABAoICAQC4sATwP563pXTRpNYq3lCI", + "P2COyqq70/qUA0PNygYt8Nr60srz5RG0WknVvefgm2U+lvFaDsqjyzkbhsf2ndnb", + "aWH/07BLdeluGB/5W2rvDFtJIVVlSmF8OffgJgohzbpjkPrfglKWMkz5LbwwrrD0", + "w0mAUIdB+nYqBfnvlKjNKHCSc9hJU6ZTNg0K7gCfKgUWzOpFlvJ0fp7XSZPYZHL0", + "2E6e0Y0Ig0cPBPb9r4/xoe+hRsHtUafUVik3PK+1K0K0FurUQ9VkQ2yUEg83F0v8", + "Vzht5OuaRVSB+P8O/JtIamfywAY0YOYhepQhjWikwU5UUzhJ+PqNDD87/+g9bA1B", + "xC25eoDxThiQlgDmRoH18ZsWDVf9TuJnm4cpxnZYX6ip+BLm/aidT39auZo0Fl+r", + "cJxRn0Qlm0Vm4Tc/6ZG6PQWB+Q6CjVFdoxeOvEQcTSuKA6VZBStLmqX++5In1Lmj", + "hVr3/aueHiZvXS5bNIdd2IfzatR+nP+uxzM/ryJRvGO2B2XTS00Cvv/lH84BDJYV", + "yt1PJIBoM9Dh7aUAHmKNVfRt83xzvcSPZx9VmSzA6wwqCQcO1GJk6keAuxOuligu", + "YdSFcfChOg90WvBcl+NzMblLkwrFSBQR7kgG0+dedv+Wkm4xO4T7B4W2G5+VIJKG", + "mrEAq6XQMFnfEJzNVg7JUQKCAQEA91eMvphoVVz+cxa4Ew7OokNXk5kSlvmQ8+Ij", + "ngFBvniXPZecxVzFEJglSthH5KI2ZqxwF3GJhKjxUihwf6K13Hx54EM7S/qV57ie", + "kVeKdAs+SGv+hRk1gQOoPBInbtKGKTni1V8T7iNginLueC/YikFugzv6IxiliBSG", + "3R7zjRepOW69aEoCPecx9amU4CkAwgeLJgBloBoqWD8sKM+bl7p5juQCU2sQ9D4/", + "kLnpG9+zPRUNjI4sog3L1wql3zthI6/4gf0TNuDhJTZ68vpMSi02pOUkVa0MmVOA", + "ex16luIp0BhxG/sUAeoevFL4KqR0CBbyAstbt2E/oPYOWMJ4MwKCAQEA0YMNXY7I", + "RNFOmiZ2Wn8kENCyJguqbOMd/li2+ercgp3MaSgTjC5KDFvZqTwXXlrURPu9hcyv", + "sJBSjp45g1T1LsUUq8UJgOIQgxykurIstGg33TAR+TN9VUu/xonLQF23GD8M6Vzd", + "EcZEVlBY33hgNXw4mRcBPnaoG5FZCBfHOgdBCExoYKW/RNKcmu0q+h9uhDBCbopv", + "04ROzw+HW1qc5qvNPR47buZ9+5QdonVK8s2bguMJ0phXwdSxL21wsjIsXyAO9m7w", + "qLHOq/hVokM0Fki09Exg4ppB8cLHC2ITpsVSgn4Dcz5zRtyvhozSKX4R9kMC64a0", + "AgMPVMllmGlR1QKCAQBIIGCrh7gNBIQyvXJKJGw/RxH3uZCBNB9/7vbh8Y3hZDr+", + "PAL8TpQsxaFCVRWJ53+jqy84tuQaKkXM5nv/zEvqEuZbbl+RRW6HVv/udC+srUap", + "Scy7tWEz0QQzGDwlhgCXbwjlnccrD2fsl51QsOsdTf1TCZ9ksqA6sXmua4MsJrUz", + "SUa0bbh/oraf46bFQ0+0RQzftQftixPEDg/rirbdpQQjlfvTpYoZHzncE0qV1ULo", + "UgZUcXU0gH9rovBBy4gFJyB5j3oV67fb6SorRrAOhWbE6QkSbtcYsw/pVuxTqXn1", + "89qwBSSNdl8mHa++h5xKa56BEBobvKEYaAhA+9yfAoIBAQDPFEE5n87Cdj7CjhGd", + "EN2M4Tmz8QPZ7AHRS85O5pxdXaqUpU/s1jPKU16nrwVJ9WypYkjI3q8oTP3MiQc/", + "j9FnENSFkpL6GHdJoB4Rido11myg6spZDVNr4xsCGWATlo1KIceZQHghAV66EWBG", + "QKyXMNigN+S64Hz4AomFPjtkV5cnpJ3mKO0MET9IwfIglsCdVzXSHHK7FaLvdeHL", + "oZxDQrvxFNiZnKgY6SUBVf1mT2LN06n5xSm4I4md3wXsmzrQKtefK7gihNxJjYLW", + "hqYNAIAalwOL9fwIAQTLc30I8S/EWtj+J1O5TpcO3lE7QahvR3yzXsi81Flq7ETG", + "iBKhAoIBAGHGpnjrLlCarNk9axh4Dw1OjgEvwPlEqsiWXt2tylLeab0OGC47MmJx", + "RmKwgVukMuxkQb8v4ANSRtih7R+E+qXfexjEFYtzh/uaRP1Z7ZrO/oqq0oLbPpsx", + "yTSRDL1i5/fgdIlKVH3N4IF7E8Pc3REgYIwLQxYjTdgVHEAM65XegQ2Lkpr4iae3", + "hm4IsD2PrsVITrlsLg65XnfcbsCs/OfQ5GuUp+xUBw5e0bQBmsWEiCaCjrq/EHJa", + "/oeJRqS7lyGYDC+wiSsE70x4dvu1um2F+V1Jw4LWjhu8Z8dNSXPSf8vLqXGkWAlk", + "805lq+iy7Mkhb+dlr4R9WhMWDyGwgYs=", + "-----END PRIVATE KEY-----" + ] +} diff --git a/ceph/src/cephadm/tox.ini b/ceph/src/cephadm/tox.ini index 66edfe99f..59d94a4c4 100644 --- a/ceph/src/cephadm/tox.ini +++ b/ceph/src/cephadm/tox.ini @@ -1,8 +1,8 @@ [tox] envlist = py27, py3, mypy +skipsdist=true [testenv] -skipsdist=true skip_install=true deps = pytest diff --git a/ceph/src/client/Client.cc b/ceph/src/client/Client.cc index 84fcf91d0..d3dceee2e 100644 --- a/ceph/src/client/Client.cc +++ b/ceph/src/client/Client.cc @@ -1844,7 +1844,7 @@ int Client::encode_inode_release(Inode *in, MetaRequest *req, { ldout(cct, 20) << __func__ << " enter(in:" << *in << ", req:" << req << " mds:" << mds << ", drop:" << drop << ", unless:" << unless - << ", have:" << ", force:" << force << ")" << dendl; + << ", force:" << force << ")" << dendl; int released = 0; auto it = in->caps.find(mds); if (it != in->caps.end()) { @@ -1852,15 +1852,20 @@ int Client::encode_inode_release(Inode *in, MetaRequest *req, drop &= ~(in->dirty_caps | get_caps_used(in)); if ((drop & cap.issued) && !(unless & cap.issued)) { - ldout(cct, 25) << "Dropping caps. Initial " << ccap_string(cap.issued) << dendl; + ldout(cct, 25) << "dropping caps " << ccap_string(drop) << dendl; cap.issued &= ~drop; cap.implemented &= ~drop; released = 1; - ldout(cct, 25) << "Now have: " << ccap_string(cap.issued) << dendl; } else { released = force; } if (released) { + cap.wanted = in->caps_wanted(); + if (&cap == in->auth_cap && + !(cap.wanted & CEPH_CAP_ANY_FILE_WR)) { + in->requested_max_size = 0; + ldout(cct, 25) << "reset requested_max_size due to not wanting any file write cap" << dendl; + } ceph_mds_request_release rel; rel.ino = in->ino; rel.cap_id = cap.cap_id; @@ -3234,13 +3239,16 @@ int Client::get_caps(Inode *in, int need, int want, int *phave, loff_t endoff) bool waitfor_commit = false; if (have & need & CEPH_CAP_FILE_WR) { - if (endoff > 0 && - (endoff >= (loff_t)in->max_size || - endoff > (loff_t)(in->size << 1)) && - endoff > (loff_t)in->wanted_max_size) { - ldout(cct, 10) << "wanted_max_size " << in->wanted_max_size << " -> " << endoff << dendl; - in->wanted_max_size = endoff; - check_caps(in, 0); + if (endoff > 0) { + if ((endoff >= (loff_t)in->max_size || + endoff > (loff_t)(in->size << 1)) && + endoff > (loff_t)in->wanted_max_size) { + ldout(cct, 10) << "wanted_max_size " << in->wanted_max_size << " -> " << endoff << dendl; + in->wanted_max_size = endoff; + } + if (in->wanted_max_size > in->max_size && + in->wanted_max_size > in->requested_max_size) + check_caps(in, 0); } if (endoff >= 0 && endoff > (loff_t)in->max_size) { @@ -3424,9 +3432,14 @@ void Client::send_cap(Inode *in, MetaSession *session, Cap *cap, m->set_snap_follows(follows); cap->wanted = want; if (cap == in->auth_cap) { - m->set_max_size(in->wanted_max_size); - in->requested_max_size = in->wanted_max_size; - ldout(cct, 15) << "auth cap, setting max_size = " << in->requested_max_size << dendl; + if (want & CEPH_CAP_ANY_FILE_WR) { + m->set_max_size(in->wanted_max_size); + in->requested_max_size = in->wanted_max_size; + ldout(cct, 15) << "auth cap, requesting max_size " << in->requested_max_size << dendl; + } else { + in->requested_max_size = 0; + ldout(cct, 15) << "auth cap, reset requested_max_size due to not wanting any file write cap" << dendl; + } } if (!session->flushing_caps_tids.empty()) @@ -4901,8 +4914,10 @@ void Client::handle_cap_import(MetaSession *session, Inode *in, const MConstRef< SnapRealm *realm = NULL; update_snap_trace(m->snapbl, &realm); + int issued = m->get_caps(); + int wanted = m->get_wanted(); add_update_cap(in, session, m->get_cap_id(), - m->get_caps(), m->get_wanted(), m->get_seq(), m->get_mseq(), + issued, wanted, m->get_seq(), m->get_mseq(), m->get_realm(), CEPH_CAP_FLAG_AUTH, cap_perms); if (cap && cap->cap_id == m->peer.cap_id) { @@ -4913,6 +4928,11 @@ void Client::handle_cap_import(MetaSession *session, Inode *in, const MConstRef< put_snap_realm(realm); if (in->auth_cap && in->auth_cap->session == session) { + if (!(wanted & CEPH_CAP_ANY_FILE_WR) || + in->requested_max_size > m->get_max_size()) { + in->requested_max_size = 0; + ldout(cct, 15) << "reset requested_max_size after cap import" << dendl; + } // reflush any/all caps (if we are now the auth_cap) kick_flushing_caps(in, session); } diff --git a/ceph/src/client/fuse_ll.cc b/ceph/src/client/fuse_ll.cc index e21f27930..3aeda459c 100644 --- a/ceph/src/client/fuse_ll.cc +++ b/ceph/src/client/fuse_ll.cc @@ -85,9 +85,13 @@ public: int fd_on_success; Client *client; - struct fuse_chan *ch; struct fuse_session *se; +#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0) + struct fuse_cmdline_opts opts; +#else + struct fuse_chan *ch; char *mountpoint; +#endif ceph::mutex stag_lock = ceph::make_mutex("fuse_ll.cc stag_lock"); int last_stag; @@ -418,7 +422,11 @@ static void fuse_ll_mkdir(fuse_req_t req, fuse_ino_t parent, const char *name, if (cfuse->fino_snap(parent) == CEPH_SNAPDIR && fuse_multithreaded && fuse_syncfs_on_mksnap) { int err = 0; +#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0) + int fd = ::open(cfuse->opts.mountpoint, O_RDONLY | O_DIRECTORY | O_CLOEXEC); +#else int fd = ::open(cfuse->mountpoint, O_RDONLY | O_DIRECTORY | O_CLOEXEC); +#endif if (fd < 0) { err = errno; } else { @@ -504,7 +512,11 @@ static void fuse_ll_symlink(fuse_req_t req, const char *existing, } static void fuse_ll_rename(fuse_req_t req, fuse_ino_t parent, const char *name, - fuse_ino_t newparent, const char *newname) + fuse_ino_t newparent, const char *newname +#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0) + , unsigned int flags +#endif + ) { CephFuse::Handle *cfuse = fuse_ll_req_prepare(req); const struct fuse_ctx *ctx = fuse_req_ctx(req); @@ -916,8 +928,12 @@ static void ino_invalidate_cb(void *handle, vinodeno_t vino, int64_t off, #if FUSE_VERSION >= FUSE_MAKE_VERSION(2, 8) CephFuse::Handle *cfuse = (CephFuse::Handle *)handle; fuse_ino_t fino = cfuse->make_fake_ino(vino.ino, vino.snapid); +#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0) + fuse_lowlevel_notify_inval_inode(cfuse->se, fino, off, len); +#else fuse_lowlevel_notify_inval_inode(cfuse->ch, fino, off, len); #endif +#endif } static void dentry_invalidate_cb(void *handle, vinodeno_t dirino, @@ -929,7 +945,11 @@ static void dentry_invalidate_cb(void *handle, vinodeno_t dirino, fuse_ino_t fino = 0; if (ino.ino != inodeno_t()) fino = cfuse->make_fake_ino(ino.ino, ino.snapid); +#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0) + fuse_lowlevel_notify_delete(cfuse->se, fdirino, fino, name.c_str(), name.length()); +#else fuse_lowlevel_notify_delete(cfuse->ch, fdirino, fino, name.c_str(), name.length()); +#endif #elif FUSE_VERSION >= FUSE_MAKE_VERSION(2, 8) fuse_lowlevel_notify_inval_entry(cfuse->ch, fdirino, name.c_str(), name.length()); #endif @@ -941,7 +961,12 @@ static int remount_cb(void *handle) // trims all unused dentries in the file system char cmd[128+PATH_MAX]; CephFuse::Handle *cfuse = (CephFuse::Handle *)handle; - snprintf(cmd, sizeof(cmd), "mount -i -o remount %s", cfuse->mountpoint); + snprintf(cmd, sizeof(cmd), "LIBMOUNT_FSTAB=/dev/null mount -i -o remount %s", +#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0) + cfuse->opts.mountpoint); +#else + cfuse->mountpoint); +#endif int r = system(cmd); if (r != 0 && r != -1) { r = WEXITSTATUS(r); @@ -1043,14 +1068,19 @@ const static struct fuse_lowlevel_ops fuse_ll_oper = { CephFuse::Handle::Handle(Client *c, int fd) : fd_on_success(fd), client(c), - ch(NULL), se(NULL), +#if FUSE_VERSION < FUSE_MAKE_VERSION(3, 0) + ch(NULL), mountpoint(NULL), +#endif last_stag(0) { snap_stag_map[CEPH_NOSNAP] = 0; stag_snap_map[0] = CEPH_NOSNAP; memset(&args, 0, sizeof(args)); +#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0) + memset(&opts, 0, sizeof(opts)); +#endif } CephFuse::Handle::~Handle() @@ -1060,6 +1090,15 @@ CephFuse::Handle::~Handle() void CephFuse::Handle::finalize() { +#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0) + if (se) { + fuse_remove_signal_handlers(se); + fuse_session_unmount(se); + fuse_session_destroy(se); + } + if (opts.mountpoint) + free(opts.mountpoint); +#else if (se) fuse_remove_signal_handlers(se); if (ch) @@ -1068,6 +1107,7 @@ void CephFuse::Handle::finalize() fuse_session_destroy(se); if (ch) fuse_unmount(mountpoint, ch); +#endif pthread_key_delete(fuse_req_key); } @@ -1094,14 +1134,16 @@ int CephFuse::Handle::init(int argc, const char *argv[]) "fuse_allow_other"); auto fuse_default_permissions = client->cct->_conf.get_val( "fuse_default_permissions"); +#if FUSE_VERSION < FUSE_MAKE_VERSION(3, 0) auto fuse_big_writes = client->cct->_conf.get_val( "fuse_big_writes"); + auto fuse_max_write = client->cct->_conf.get_val( + "fuse_max_write"); auto fuse_atomic_o_trunc = client->cct->_conf.get_val( "fuse_atomic_o_trunc"); +#endif auto fuse_debug = client->cct->_conf.get_val( "fuse_debug"); - auto fuse_max_write = client->cct->_conf.get_val( - "fuse_max_write"); if (fuse_allow_other) { newargv[newargc++] = "-o"; @@ -1112,6 +1154,7 @@ int CephFuse::Handle::init(int argc, const char *argv[]) newargv[newargc++] = "default_permissions"; } #if defined(__linux__) +#if FUSE_VERSION < FUSE_MAKE_VERSION(3, 0) if (fuse_big_writes) { newargv[newargc++] = "-o"; newargv[newargc++] = "big_writes"; @@ -1127,6 +1170,7 @@ int CephFuse::Handle::init(int argc, const char *argv[]) newargv[newargc++] = "-o"; newargv[newargc++] = "atomic_o_trunc"; } +#endif #endif if (fuse_debug) newargv[newargc++] = "-d"; @@ -1138,7 +1182,11 @@ int CephFuse::Handle::init(int argc, const char *argv[]) struct fuse_args a = FUSE_ARGS_INIT(newargc, (char**)newargv); args = a; // Roundabout construction b/c FUSE_ARGS_INIT is for initialization not assignment +#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0) + if (fuse_parse_cmdline(&args, &opts) == -1) { +#else if (fuse_parse_cmdline(&args, &mountpoint, NULL, NULL) == -1) { +#endif derr << "fuse_parse_cmdline failed." << dendl; fuse_opt_free_args(&args); free(newargv); @@ -1152,6 +1200,9 @@ int CephFuse::Handle::init(int argc, const char *argv[]) int CephFuse::Handle::start() { +#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0) + se = fuse_session_new(&args, &fuse_ll_oper, sizeof(fuse_ll_oper), this); +#else ch = fuse_mount(mountpoint, &args); if (!ch) { derr << "fuse_mount(mountpoint=" << mountpoint << ") failed." << dendl; @@ -1159,6 +1210,7 @@ int CephFuse::Handle::start() } se = fuse_lowlevel_new(&args, &fuse_ll_oper, sizeof(fuse_ll_oper), this); +#endif if (!se) { derr << "fuse_lowlevel_new failed" << dendl; return EDOM; @@ -1171,7 +1223,14 @@ int CephFuse::Handle::start() return ENOSYS; } +#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0) + if (fuse_session_mount(se, opts.mountpoint) != 0) { + derr << "fuse_session_mount failed" << dendl; + return ENOSYS; + } +#else fuse_session_add_chan(se, ch); +#endif struct client_callback_args args = { @@ -1197,7 +1256,11 @@ int CephFuse::Handle::loop() auto fuse_multithreaded = client->cct->_conf.get_val( "fuse_multithreaded"); if (fuse_multithreaded) { +#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0) + return fuse_session_loop_mt(se, opts.clone_fd); +#else return fuse_session_loop_mt(se); +#endif } else { return fuse_session_loop(se); } @@ -1327,8 +1390,13 @@ void CephFuse::finalize() std::string CephFuse::get_mount_point() const { +#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0) + if (_handle->opts.mountpoint) { + return _handle->opts.mountpoint; +#else if (_handle->mountpoint) { return _handle->mountpoint; +#endif } else { return ""; } diff --git a/ceph/src/cls/journal/cls_journal_types.h b/ceph/src/cls/journal/cls_journal_types.h index 705936a06..f82d30c7e 100644 --- a/ceph/src/cls/journal/cls_journal_types.h +++ b/ceph/src/cls/journal/cls_journal_types.h @@ -35,6 +35,9 @@ struct ObjectPosition { tag_tid == rhs.tag_tid && entry_tid == rhs.entry_tid); } + inline bool operator!=(const ObjectPosition& rhs) const { + return !(*this == rhs); + } void encode(ceph::buffer::list& bl) const; void decode(ceph::buffer::list::const_iterator& iter); diff --git a/ceph/src/cls/rbd/cls_rbd_types.cc b/ceph/src/cls/rbd/cls_rbd_types.cc index 9071f3457..5f3c40b04 100644 --- a/ceph/src/cls/rbd/cls_rbd_types.cc +++ b/ceph/src/cls/rbd/cls_rbd_types.cc @@ -767,10 +767,14 @@ void MirrorSnapshotNamespace::dump(Formatter *f) const { f->dump_string("mirror_peer_uuid", peer); } f->close_section(); - f->dump_string("primary_mirror_uuid", primary_mirror_uuid); - f->dump_unsigned("primary_snap_id", primary_snap_id); - f->dump_unsigned("last_copied_object_number", last_copied_object_number); - f->dump_stream("snap_seqs") << snap_seqs; + if (is_primary()) { + f->dump_unsigned("clean_since_snap_id", clean_since_snap_id); + } else { + f->dump_string("primary_mirror_uuid", primary_mirror_uuid); + f->dump_unsigned("primary_snap_id", primary_snap_id); + f->dump_unsigned("last_copied_object_number", last_copied_object_number); + f->dump_stream("snap_seqs") << snap_seqs; + } } class EncodeSnapshotNamespaceVisitor : public boost::static_visitor { diff --git a/ceph/src/cls/rbd/cls_rbd_types.h b/ceph/src/cls/rbd/cls_rbd_types.h index b2c155eb8..5ea37caf7 100644 --- a/ceph/src/cls/rbd/cls_rbd_types.h +++ b/ceph/src/cls/rbd/cls_rbd_types.h @@ -545,7 +545,10 @@ struct MirrorSnapshotNamespace { std::set mirror_peer_uuids; std::string primary_mirror_uuid; - snapid_t primary_snap_id = CEPH_NOSNAP; + union { + snapid_t primary_snap_id = CEPH_NOSNAP; + snapid_t clean_since_snap_id; + }; uint64_t last_copied_object_number = 0; SnapSeqs snap_seqs; diff --git a/ceph/src/common/AsyncOpTracker.h b/ceph/src/common/AsyncOpTracker.h index d913032aa..dfa913ad4 100644 --- a/ceph/src/common/AsyncOpTracker.h +++ b/ceph/src/common/AsyncOpTracker.h @@ -5,8 +5,7 @@ #define CEPH_ASYNC_OP_TRACKER_H #include "common/ceph_mutex.h" - -struct Context; +#include "include/Context.h" class AsyncOpTracker { public: @@ -27,4 +26,23 @@ private: }; +class C_TrackedOp : public Context { +public: + C_TrackedOp(AsyncOpTracker& async_op_tracker, Context* on_finish) + : m_async_op_tracker(async_op_tracker), m_on_finish(on_finish) { + m_async_op_tracker.start_op(); + } + + void finish(int r) override { + if (m_on_finish != nullptr) { + m_on_finish->complete(r); + } + m_async_op_tracker.finish_op(); + } + +private: + AsyncOpTracker& m_async_op_tracker; + Context* m_on_finish; +}; + #endif // CEPH_ASYNC_OP_TRACKER_H diff --git a/ceph/src/common/options.cc b/ceph/src/common/options.cc index 1354b133f..bb3b3ef39 100644 --- a/ceph/src/common/options.cc +++ b/ceph/src/common/options.cc @@ -3995,8 +3995,9 @@ std::vector