From e306af509c4d4816a1f73b17a825ea5186fa0030 Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Thu, 2 Jul 2020 15:37:13 +0200 Subject: [PATCH] import 15.2.4 Signed-off-by: Thomas Lamprecht --- ceph/CMakeLists.txt | 2 +- ceph/PendingReleaseNotes | 369 +- ceph/alpine/APKBUILD | 8 +- ceph/alpine/APKBUILD.in | 2 +- ceph/ceph.spec | 14 +- ceph/ceph.spec.in | 8 +- ceph/changelog.upstream | 10 +- ceph/cmake/modules/FindFUSE.cmake | 55 +- ceph/debian/cephadm.postinst | 2 +- ceph/debian/control | 3 +- ceph/debian/libcephfs-dev.install | 2 +- ceph/debian/radosgw.install | 4 + ceph/doc/cephadm/adoption.rst | 3 + ceph/doc/cephadm/install.rst | 15 +- ceph/doc/cephadm/monitoring.rst | 75 +- ceph/doc/cephadm/operations.rst | 20 + ceph/doc/cephadm/troubleshooting.rst | 28 + ceph/doc/cephfs/administration.rst | 3 + ceph/doc/cephfs/createfs.rst | 2 + ceph/doc/cephfs/file-layouts.rst | 4 +- ceph/doc/cephfs/fs-volumes.rst | 20 +- ceph/doc/dev/cephadm.rst | 15 + .../images/dashboard/invalid-credentials.png | Bin 0 -> 5787 bytes ceph/doc/install/containers.rst | 13 + ceph/doc/man/8/CMakeLists.txt | 4 +- ceph/doc/man/8/ceph-diff-sorted.rst | 71 + ceph/doc/man/8/ceph-osd.rst | 6 +- ceph/doc/man/8/ceph.rst | 3 +- ceph/doc/man/8/cephadm.rst | 14 +- ceph/doc/man/8/radosgw-admin.rst | 12 +- ceph/doc/man/8/rgw-orphan-list.rst | 69 + ceph/doc/man_index.rst | 3 + ceph/doc/mgr/dashboard.rst | 199 +- ceph/doc/mgr/orchestrator.rst | 181 +- ceph/doc/mgr/prometheus.rst | 10 + ceph/doc/mgr/telemetry.rst | 11 +- ceph/doc/rados/operations/devices.rst | 21 + ceph/doc/rados/operations/monitoring.rst | 1 + ceph/doc/radosgw/config-ref.rst | 9 + ceph/doc/radosgw/index.rst | 1 + ceph/doc/radosgw/multisite.rst | 2 + ceph/doc/radosgw/notifications.rst | 5 +- ceph/doc/radosgw/orphans.rst | 115 + ceph/doc/radosgw/pubsub-module.rst | 10 +- ceph/fusetrace/fusetrace_ll.cc | 2 - .../dashboards/radosgw-sync-overview.json | 440 + .../grafana/dashboards/rbd-details.json | 409 + .../prometheus/alerts/ceph_default_alerts.yml | 4 +- ceph/qa/.teuthology_branch | 1 + ...uestore-avl.yaml => bluestore-hybrid.yaml} | 4 +- ceph/qa/standalone/scrub/osd-scrub-repair.sh | 2 +- .../client_trim_caps/tasks/trim-i22073.yaml | 1 + .../krbd/thrash/thrashers/mon-thrasher.yaml | 4 + .../powercycle/osd/powercycle/default.yaml | 20 +- .../osd/tasks/cfuse_workunit_suites_fsx.yaml | 12 - .../suites/rados/cephadm/smoke/fixed-2.yaml | 1 + .../cephadm/upgrade/2-start-upgrade.yaml | 2 +- .../suites/rados/cephadm/upgrade/fixed-2.yaml | 32 +- .../with-work/distro/ubuntu_latest.yaml | 1 - .../workunits/distro/ubuntu_latest.yaml | 1 - .../rados/dashboard/tasks/dashboard.yaml | 1 + ceph/qa/suites/rgw/crypt/2-kms/barbican.yaml | 37 +- ceph/qa/suites/rgw/crypt/4-tests/s3tests.yaml | 2 +- .../suites/rgw/multifs/tasks/rgw_ragweed.yaml | 4 +- .../suites/rgw/multifs/tasks/rgw_s3tests.yaml | 2 +- .../suites/rgw/tempest/tasks/rgw_tempest.yaml | 45 +- .../rgw/thrash/workload/rgw_s3tests.yaml | 2 +- ceph/qa/suites/rgw/tools/+ | 0 ceph/qa/suites/rgw/tools/.qa | 1 + ceph/qa/suites/rgw/tools/centos_latest.yaml | 1 + ceph/qa/suites/rgw/tools/cluster.yaml | 9 + ceph/qa/suites/rgw/tools/tasks.yaml | 19 + ceph/qa/suites/rgw/verify/tasks/ragweed.yaml | 2 +- ceph/qa/suites/rgw/verify/tasks/s3tests.yaml | 2 +- .../rgw/website/tasks/s3tests-website.yaml | 2 +- .../smoke/basic/tasks/rgw_ec_s3tests.yaml | 2 +- .../suites/smoke/basic/tasks/rgw_s3tests.yaml | 2 +- .../point-to-point-upgrade.yaml | 25 +- ceph/qa/tasks/barbican.py | 102 +- ceph/qa/tasks/cbt.py | 8 +- ceph/qa/tasks/ceph_fuse.py | 2 +- ceph/qa/tasks/ceph_manager.py | 19 +- ceph/qa/tasks/ceph_objectstore_tool.py | 31 +- ceph/qa/tasks/cephadm.py | 292 +- ceph/qa/tasks/cephfs/cephfs_test_case.py | 32 +- ceph/qa/tasks/cephfs/filesystem.py | 32 +- ceph/qa/tasks/cephfs/fuse_mount.py | 76 +- ceph/qa/tasks/cephfs/kernel_mount.py | 15 +- ceph/qa/tasks/cephfs/mount.py | 18 +- ceph/qa/tasks/cephfs/test_acls.py | 8 +- ceph/qa/tasks/cephfs/test_admin.py | 15 + ceph/qa/tasks/cephfs/test_auto_repair.py | 3 +- ceph/qa/tasks/cephfs/test_cephfs_shell.py | 51 +- ceph/qa/tasks/cephfs/test_client_limits.py | 15 +- ceph/qa/tasks/cephfs/test_client_recovery.py | 20 +- ceph/qa/tasks/cephfs/test_damage.py | 11 +- ceph/qa/tasks/cephfs/test_data_scan.py | 31 +- ceph/qa/tasks/cephfs/test_dump_tree.py | 2 +- ceph/qa/tasks/cephfs/test_exports.py | 11 +- ceph/qa/tasks/cephfs/test_failover.py | 10 +- ceph/qa/tasks/cephfs/test_flush.py | 3 +- ceph/qa/tasks/cephfs/test_forward_scrub.py | 16 +- ceph/qa/tasks/cephfs/test_full.py | 12 +- .../qa/tasks/cephfs/test_journal_migration.py | 9 +- ceph/qa/tasks/cephfs/test_journal_repair.py | 10 +- ceph/qa/tasks/cephfs/test_misc.py | 5 +- ceph/qa/tasks/cephfs/test_openfiletable.py | 4 +- ceph/qa/tasks/cephfs/test_pool_perm.py | 12 +- ceph/qa/tasks/cephfs/test_quota.py | 2 +- ceph/qa/tasks/cephfs/test_readahead.py | 3 +- ceph/qa/tasks/cephfs/test_recovery_pool.py | 6 +- ceph/qa/tasks/cephfs/test_scrub.py | 30 +- ceph/qa/tasks/cephfs/test_scrub_checks.py | 3 +- ceph/qa/tasks/cephfs/test_sessionmap.py | 13 +- ceph/qa/tasks/cephfs/test_snapshots.py | 14 +- ceph/qa/tasks/cephfs/test_strays.py | 15 +- ceph/qa/tasks/cephfs/test_volume_client.py | 12 +- ceph/qa/tasks/cephfs/test_volumes.py | 108 +- ceph/qa/tasks/cephfs/xfstests_dev.py | 8 +- ceph/qa/tasks/check_counter.py | 2 +- ceph/qa/tasks/cram.py | 2 +- ceph/qa/tasks/devstack.py | 46 +- ceph/qa/tasks/die_on_err.py | 2 +- ceph/qa/tasks/divergent_priors.py | 2 +- ceph/qa/tasks/divergent_priors2.py | 2 +- ceph/qa/tasks/dnsmasq.py | 4 +- ceph/qa/tasks/dump_stuck.py | 2 +- ceph/qa/tasks/ec_lost_unfound.py | 8 +- ceph/qa/tasks/filestore_idempotent.py | 2 +- ceph/qa/tasks/fs.py | 5 +- ceph/qa/tasks/kclient.py | 4 +- ceph/qa/tasks/keystone.py | 130 +- ceph/qa/tasks/lost_unfound.py | 4 +- ceph/qa/tasks/mds_creation_failure.py | 2 +- ceph/qa/tasks/mds_thrash.py | 19 +- ceph/qa/tasks/mgr/dashboard/helper.py | 2 +- ceph/qa/tasks/mgr/dashboard/test_auth.py | 2 +- ceph/qa/tasks/mgr/dashboard/test_cephfs.py | 2 +- .../dashboard/test_cluster_configuration.py | 2 +- .../qa/tasks/mgr/dashboard/test_crush_rule.py | 2 +- .../dashboard/test_erasure_code_profile.py | 5 +- ceph/qa/tasks/mgr/dashboard/test_ganesha.py | 2 +- ceph/qa/tasks/mgr/dashboard/test_health.py | 2 +- ceph/qa/tasks/mgr/dashboard/test_host.py | 4 +- ceph/qa/tasks/mgr/dashboard/test_logs.py | 2 +- .../qa/tasks/mgr/dashboard/test_mgr_module.py | 2 +- ceph/qa/tasks/mgr/dashboard/test_monitor.py | 2 +- .../tasks/mgr/dashboard/test_orchestrator.py | 2 +- ceph/qa/tasks/mgr/dashboard/test_osd.py | 2 +- .../tasks/mgr/dashboard/test_perf_counters.py | 2 +- ceph/qa/tasks/mgr/dashboard/test_pool.py | 3 +- ceph/qa/tasks/mgr/dashboard/test_rbd.py | 2 +- .../tasks/mgr/dashboard/test_rbd_mirroring.py | 2 +- ceph/qa/tasks/mgr/dashboard/test_requests.py | 2 +- ceph/qa/tasks/mgr/dashboard/test_rgw.py | 33 +- ceph/qa/tasks/mgr/dashboard/test_role.py | 2 +- ceph/qa/tasks/mgr/dashboard/test_settings.py | 2 +- ceph/qa/tasks/mgr/dashboard/test_summary.py | 2 +- ceph/qa/tasks/mgr/dashboard/test_telemetry.py | 89 + ceph/qa/tasks/mgr/dashboard/test_user.py | 2 +- ceph/qa/tasks/mgr/mgr_test_case.py | 2 +- ceph/qa/tasks/mgr/test_crash.py | 4 +- ceph/qa/tasks/mgr/test_dashboard.py | 2 +- ceph/qa/tasks/mgr/test_insights.py | 3 +- ceph/qa/tasks/mgr/test_orchestrator_cli.py | 2 +- ceph/qa/tasks/mgr/test_progress.py | 2 +- ceph/qa/tasks/mgr/test_prometheus.py | 4 +- ceph/qa/tasks/mon_clock_skew_check.py | 2 +- ceph/qa/tasks/mon_recovery.py | 4 +- ceph/qa/tasks/multibench.py | 3 +- ceph/qa/tasks/object_source_down.py | 4 +- ceph/qa/tasks/openssl_keys.py | 5 +- ceph/qa/tasks/osd_backfill.py | 2 +- ceph/qa/tasks/osd_failsafe_enospc.py | 2 +- ceph/qa/tasks/osd_recovery.py | 2 +- ceph/qa/tasks/peer.py | 4 +- ceph/qa/tasks/peering_speed_test.py | 2 +- ceph/qa/tasks/qemu.py | 16 +- ceph/qa/tasks/rados.py | 8 +- ceph/qa/tasks/radosbench.py | 23 +- ceph/qa/tasks/radosgw_admin.py | 43 +- ceph/qa/tasks/radosgw_admin_rest.py | 4 +- ceph/qa/tasks/ragweed.py | 17 +- ceph/qa/tasks/rbd.py | 9 +- ceph/qa/tasks/rbd_fio.py | 14 +- ceph/qa/tasks/rbd_mirror.py | 2 +- ceph/qa/tasks/rebuild_mondb.py | 2 +- ceph/qa/tasks/reg11184.py | 2 +- ceph/qa/tasks/rep_lost_unfound_delete.py | 7 +- ceph/qa/tasks/resolve_stuck_peering.py | 2 +- ceph/qa/tasks/rgw.py | 19 +- ceph/qa/tasks/rgw_logsocket.py | 6 +- ceph/qa/tasks/rgw_multisite.py | 10 +- ceph/qa/tasks/rgw_multisite_tests.py | 2 +- ceph/qa/tasks/s3tests.py | 23 +- ceph/qa/tasks/s3tests_java.py | 34 +- ceph/qa/tasks/scrub_test.py | 2 +- ceph/qa/tasks/tempest.py | 32 +- ceph/qa/tasks/tests/test_cephadm.py | 70 + ceph/qa/tasks/tests/test_devstack.py | 2 +- ceph/qa/tasks/tests/test_radosgw_admin.py | 8 +- ceph/qa/tasks/thrashosds.py | 2 +- ceph/qa/tasks/tox.py | 4 +- ceph/qa/tasks/util/rgw.py | 3 +- ceph/qa/tasks/util/test/test_rados.py | 2 +- ceph/qa/tasks/vault.py | 10 +- ceph/qa/tasks/vstart_runner.py | 38 +- ceph/qa/tasks/watch_notify_same_primary.py | 6 +- ceph/qa/tasks/watch_notify_stress.py | 2 +- ceph/qa/tox.ini | 10 +- ceph/qa/valgrind.supp | 4 +- ceph/qa/workunits/ceph-helpers-root.sh | 9 - ceph/qa/workunits/cephadm/test_adoption.sh | 2 +- ceph/qa/workunits/cephadm/test_cephadm.sh | 7 +- ceph/qa/workunits/cephtool/test.sh | 21 +- .../rados/test_envlibrados_for_rocksdb.sh | 16 +- ceph/qa/workunits/rbd/rbd-nbd.sh | 2 +- ceph/qa/workunits/rgw/test_rgw_orphan_list.sh | 505 ++ ceph/selinux/ceph.te | 2 + ceph/src/.git_version | 4 +- ceph/src/bash_completion/ceph | 2 +- ceph/src/ceph-volume/ceph_volume/api/lvm.py | 30 +- .../ceph_volume/devices/lvm/prepare.py | 1 + .../ceph_volume/tests/api/test_lvm.py | 30 +- .../ceph-volume/ceph_volume/util/prepare.py | 10 + ceph/src/ceph.in | 3 + ceph/src/ceph_fuse.cc | 2 +- ceph/src/ceph_osd.cc | 9 +- ceph/src/cephadm/cephadm | 344 +- ceph/src/cephadm/mypy.ini | 2 +- ceph/src/cephadm/tests/test_cephadm.py | 1 + ceph/src/client/Client.cc | 76 +- ceph/src/client/Client.h | 35 +- ceph/src/client/Delegation.h | 6 +- ceph/src/client/SyntheticClient.cc | 2 +- ceph/src/client/fuse_ll.cc | 31 +- ceph/src/client/fuse_ll.h | 2 - ceph/src/cls/queue/cls_queue_src.cc | 13 + ceph/src/cls/rgw_gc/cls_rgw_gc.cc | 2 + ceph/src/common/config.cc | 1 + ceph/src/common/legacy_config_opts.h | 2 + ceph/src/common/options.cc | 39 +- ceph/src/crimson/os/alienstore/CMakeLists.txt | 1 + ceph/src/include/ceph_fuse.h | 23 +- .../cephfs/{ceph_statx.h => ceph_ll_client.h} | 73 +- ceph/src/include/cephfs/libcephfs.h | 33 +- ceph/src/include/config-h.in.cmake | 6 + ceph/src/libcephfs.cc | 8 +- ceph/src/librbd/ImageCtx.cc | 4 +- ceph/src/librbd/ImageCtx.h | 4 +- ceph/src/librbd/Watcher.cc | 2 +- ceph/src/librbd/internal.cc | 2 +- ceph/src/librbd/mirror/DisableRequest.cc | 15 +- ceph/src/mds/CInode.cc | 25 +- ceph/src/mds/CInode.h | 6 + ceph/src/mds/Locker.cc | 5 +- ceph/src/mds/LogSegment.h | 3 +- ceph/src/mds/MDCache.cc | 89 +- ceph/src/mds/MDCache.h | 31 +- ceph/src/mds/MDSContext.cc | 32 +- ceph/src/mds/MDSContext.h | 5 +- ceph/src/mds/MDSRank.cc | 8 + ceph/src/mds/Mutation.h | 8 +- ceph/src/mds/PurgeQueue.cc | 8 +- ceph/src/mds/Server.cc | 16 +- ceph/src/mds/journal.cc | 39 +- ceph/src/mds/mdstypes.h | 18 +- ceph/src/mgr/ActivePyModules.cc | 25 +- ceph/src/mgr/ClusterState.cc | 1 + ceph/src/mgr/ClusterState.h | 18 +- ceph/src/mon/MonCommands.h | 2 +- ceph/src/mon/OSDMonitor.cc | 2 - ceph/src/os/CMakeLists.txt | 1 + ceph/src/os/FuseStore.cc | 4 +- ceph/src/os/bluestore/Allocator.cc | 9 + ceph/src/os/bluestore/Allocator.h | 3 + ceph/src/os/bluestore/AvlAllocator.cc | 258 +- ceph/src/os/bluestore/AvlAllocator.h | 147 +- ceph/src/os/bluestore/BitmapAllocator.h | 3 +- ceph/src/os/bluestore/HybridAllocator.cc | 222 + ceph/src/os/bluestore/HybridAllocator.h | 48 + ceph/src/os/bluestore/KernelDevice.cc | 40 +- .../os/bluestore/fastbmap_allocator_impl.cc | 83 + .../os/bluestore/fastbmap_allocator_impl.h | 56 +- ceph/src/osd/OSD.cc | 16 +- ceph/src/osd/OSD.h | 4 +- ceph/src/osdc/Filer.cc | 11 +- ceph/src/osdc/Filer.h | 2 +- ceph/src/osdc/Objecter.cc | 17 +- ceph/src/powerdns/pdns-backend-rgw.py | 2 +- ceph/src/pybind/ceph_volume_client.py | 10 +- ceph/src/pybind/cephfs/cephfs.pyx | 4 +- ceph/src/pybind/mgr/alerts/module.py | 2 +- ceph/src/pybind/mgr/balancer/module.py | 4 +- ceph/src/pybind/mgr/cephadm/__init__.py | 1 + ceph/src/pybind/mgr/cephadm/inventory.py | 430 + ceph/src/pybind/mgr/cephadm/module.py | 2396 ++---- ceph/src/pybind/mgr/cephadm/osd.py | 191 - ceph/src/pybind/mgr/cephadm/schedule.py | 193 + .../pybind/mgr/cephadm/services/__init__.py | 0 .../mgr/cephadm/services/cephadmservice.py | 228 + ceph/src/pybind/mgr/cephadm/services/iscsi.py | 99 + .../pybind/mgr/cephadm/services/monitoring.py | 235 + .../pybind/mgr/cephadm/{ => services}/nfs.py | 117 +- ceph/src/pybind/mgr/cephadm/services/osd.py | 430 + ceph/src/pybind/mgr/cephadm/template.py | 73 + .../services/alertmanager/alertmanager.yml.j2 | 18 + .../services/grafana/ceph-dashboard.yml.j2 | 18 + .../templates/services/grafana/grafana.ini.j2 | 17 + .../services/iscsi/iscsi-gateway.cfg.j2 | 13 + .../templates/services/nfs/ganesha.conf.j2 | 36 + .../services/prometheus/prometheus.yml.j2 | 32 + ceph/src/pybind/mgr/cephadm/tests/fixtures.py | 36 +- .../pybind/mgr/cephadm/tests/test_cephadm.py | 178 +- .../mgr/cephadm/tests/test_completion.py | 139 +- .../mgr/cephadm/tests/test_scheduling.py | 9 +- .../pybind/mgr/cephadm/tests/test_services.py | 33 + .../src/pybind/mgr/cephadm/tests/test_spec.py | 207 +- .../pybind/mgr/cephadm/tests/test_template.py | 33 + ceph/src/pybind/mgr/cephadm/upgrade.py | 378 + ceph/src/pybind/mgr/cephadm/utils.py | 15 + ceph/src/pybind/mgr/dashboard/.pylintrc | 1 - ceph/src/pybind/mgr/dashboard/CMakeLists.txt | 2 +- ceph/src/pybind/mgr/dashboard/HACKING.rst | 243 +- ceph/src/pybind/mgr/dashboard/__init__.py | 9 +- ceph/src/pybind/mgr/dashboard/conftest.py | 39 - .../mgr/dashboard/controllers/__init__.py | 4 +- .../controllers/erasure_code_profile.py | 12 +- .../pybind/mgr/dashboard/controllers/host.py | 56 +- .../pybind/mgr/dashboard/controllers/pool.py | 14 +- .../pybind/mgr/dashboard/controllers/rgw.py | 8 +- .../mgr/dashboard/controllers/telemetry.py | 43 + .../pybind/mgr/dashboard/frontend/.prettierrc | 3 +- .../mgr/dashboard/frontend/angular.json | 26 - .../mgr/dashboard/frontend/cypress.json | 14 + .../integration/block/images.e2e-spec.ts | 92 + .../cypress/integration/block/images.po.ts | 110 + .../integration/block/iscsi.e2e-spec.ts | 24 + .../integration}/block/iscsi.po.ts | 4 +- .../integration/block/mirroring.e2e-spec.ts | 53 + .../cypress/integration/block/mirroring.po.ts | 32 + .../cluster/configuration.e2e-spec.ts | 66 + .../integration/cluster/configuration.po.ts | 75 + .../integration/cluster/crush-map.e2e-spec.ts | 36 + .../integration/cluster/crush-map.po.ts | 13 + .../integration/cluster/hosts.e2e-spec.ts | 38 + .../cypress/integration/cluster/hosts.po.ts | 31 + .../integration/cluster/logs.e2e-spec.ts | 73 + .../cypress/integration/cluster/logs.po.ts | 70 + .../cluster/mgr-modules.e2e-spec.ts | 98 + .../integration/cluster/mgr-modules.po.ts | 53 + .../integration/cluster/monitors.e2e-spec.ts | 61 + .../integration}/cluster/monitors.po.ts | 4 +- .../integration/cluster/osds.e2e-spec.ts | 61 + .../integration}/cluster/osds.po.ts | 2 +- .../filesystems/filesystems.e2e-spec.ts | 16 + .../filesystems/filesystems.po.ts | 2 +- .../cypress/integration/nfs/nfs.e2e-spec.ts | 16 + .../integration}/nfs/nfs.po.ts | 2 +- .../cypress/integration/page-helper.po.ts | 259 + .../integration/pools/pools.e2e-spec.ts | 47 + .../cypress/integration/pools/pools.po.ts | 58 + .../integration/rgw/buckets.e2e-spec.ts | 56 + .../cypress/integration/rgw/buckets.po.ts | 156 + .../integration/rgw/daemons.e2e-spec.ts | 34 + .../cypress/integration/rgw/daemons.po.ts | 42 + .../cypress/integration/rgw/users.e2e-spec.ts | 43 + .../cypress/integration/rgw/users.po.ts | 130 + .../integration}/ui/dashboard.e2e-spec.ts | 106 +- .../cypress/integration/ui/dashboard.po.ts | 27 + .../integration/ui/notification.e2e-spec.ts | 56 + .../cypress/integration/ui/notification.po.ts | 45 + .../integration/ui/role-mgmt.e2e-spec.ts | 36 + .../cypress/integration/ui/role-mgmt.po.ts | 40 + .../integration/ui/user-mgmt.e2e-spec.ts | 36 + .../cypress/integration/ui/user-mgmt.po.ts | 39 + .../frontend/cypress/support/commands.ts | 47 + .../frontend/cypress/support/index.ts | 5 + .../dashboard/frontend/cypress/tsconfig.json | 13 + .../dist/en-US/2.5b0a7ba1cfec198ba8ed.js | 1 + .../dist/en-US/2.d36650ee0a92dfd05faa.js | 1 - .../frontend/dist/en-US/3rdpartylicenses.txt | 22 +- .../dist/en-US/6.4299ba94dd69b0b04046.js | 1 - .../dist/en-US/6.c0ad55977c0a220b3203.js | 1 + .../dist/en-US/7.904941689cb18cba0036.js | 1 + .../dist/en-US/7.cf5f8c70f123c771366b.js | 1 - .../dist/en-US/8.988fe96f3a1b2f8c64cd.js | 1 + .../dist/en-US/8.f45b72794d78f44d11b9.js | 1 - .../dist/en-US/9.a6e2f7400e0a62470111.js | 1 - .../dist/en-US/9.d895914fbc785284c5be.js | 1 + ...h_Logo_Standard_RGB_Reversed_120411_fa.png | Bin 0 -> 28175 bytes .../dist/en-US/assets/ceph_background.gif | Bin 0 -> 98115 bytes .../ceph_background.7c127240f022e7baf57d.gif | Bin 0 -> 98115 bytes .../dashboard/frontend/dist/en-US/index.html | 4 +- .../dist/en-US/main.a10546ecbf9c49d65e18.js | 1 + .../dist/en-US/main.bbd2d62de0d7a670cae3.js | 1 - .../en-US/runtime.2fd4e31a298ae797b302.js | 1 - .../en-US/runtime.992511a1840a2561365a.js | 1 + .../en-US/styles.4f0058c288677d311127.css | 17 + .../en-US/styles.f15465861b49727cf28a.css | 17 - .../frontend/dist/en-US/swagger-ui-bundle.js | 26 +- .../frontend/dist/en-US/swagger-ui.css | 2 +- .../frontend/e2e/block/images.e2e-spec.ts | 116 - .../dashboard/frontend/e2e/block/images.po.ts | 130 - .../frontend/e2e/block/iscsi.e2e-spec.ts | 41 - .../frontend/e2e/block/mirroring.e2e-spec.ts | 63 - .../frontend/e2e/block/mirroring.po.ts | 33 - .../e2e/cluster/configuration.e2e-spec.ts | 81 - .../frontend/e2e/cluster/configuration.po.ts | 92 - .../e2e/cluster/crush-map.e2e-spec.ts | 44 - .../frontend/e2e/cluster/crush-map.po.ts | 14 - .../frontend/e2e/cluster/hosts.e2e-spec.ts | 45 - .../frontend/e2e/cluster/hosts.po.ts | 46 - .../frontend/e2e/cluster/logs.e2e-spec.ts | 83 - .../dashboard/frontend/e2e/cluster/logs.po.ts | 134 - .../e2e/cluster/mgr-modules.e2e-spec.ts | 54 - .../frontend/e2e/cluster/mgr-modules.po.ts | 145 - .../frontend/e2e/cluster/monitors.e2e-spec.ts | 115 - .../frontend/e2e/cluster/osds.e2e-spec.ts | 71 - .../e2e/filesystems/filesystems.e2e-spec.ts | 23 - .../frontend/e2e/nfs/nfs.e2e-spec.ts | 23 - .../dashboard/frontend/e2e/page-helper.po.ts | 346 - .../frontend/e2e/pools/pools.e2e-spec.ts | 52 - .../dashboard/frontend/e2e/pools/pools.po.ts | 86 - .../frontend/e2e/rgw/buckets.e2e-spec.ts | 70 - .../dashboard/frontend/e2e/rgw/buckets.po.ts | 204 - .../frontend/e2e/rgw/daemons.e2e-spec.ts | 45 - .../dashboard/frontend/e2e/rgw/daemons.po.ts | 41 - .../frontend/e2e/rgw/users.e2e-spec.ts | 55 - .../dashboard/frontend/e2e/rgw/users.po.ts | 191 - .../dashboard/frontend/e2e/tsconfig.e2e.json | 15 - .../dashboard/frontend/e2e/ui/dashboard.po.ts | 53 - .../frontend/e2e/ui/notification.e2e-spec.ts | 73 - .../frontend/e2e/ui/notification.po.ts | 34 - .../frontend/e2e/ui/role-mgmt.e2e-spec.ts | 41 - .../dashboard/frontend/e2e/ui/role-mgmt.po.ts | 41 - .../frontend/e2e/ui/user-mgmt.e2e-spec.ts | 41 - .../dashboard/frontend/e2e/ui/user-mgmt.po.ts | 48 - .../mgr/dashboard/frontend/package-lock.json | 7212 +++++++---------- .../mgr/dashboard/frontend/package.json | 71 +- .../mgr/dashboard/frontend/protractor.conf.js | 78 - .../frontend/src/app/app-routing.module.ts | 11 +- .../dashboard/frontend/src/app/app.module.ts | 2 +- .../iscsi-target-details.component.spec.ts | 62 +- .../iscsi-target-details.component.ts | 11 +- .../iscsi-target-form.component.ts | 2 +- .../iscsi-target-list.component.html | 6 +- .../iscsi-target-list.component.spec.ts | 2 + .../iscsi-target-list.component.ts | 4 +- .../ceph/block/iscsi/iscsi.component.spec.ts | 5 +- .../daemon-list/daemon-list.component.html | 2 +- .../daemon-list/daemon-list.component.spec.ts | 2 + .../image-list/image-list.component.html | 17 +- .../image-list/image-list.component.spec.ts | 2 + .../overview/overview.component.spec.ts | 2 + .../pool-list/pool-list.component.html | 2 +- .../pool-list/pool-list.component.spec.ts | 2 + .../rbd-configuration-list.component.spec.ts | 2 + .../rbd-details/rbd-details.component.html | 54 +- .../rbd-details/rbd-details.component.ts | 14 +- .../ceph/block/rbd-form/rbd-form.component.ts | 5 +- .../block/rbd-list/rbd-list.component.html | 4 +- .../block/rbd-list/rbd-list.component.spec.ts | 2 + .../ceph/block/rbd-list/rbd-list.component.ts | 4 +- .../rbd-namespace-list.component.spec.ts | 2 + .../rbd-snapshot-list.component.spec.ts | 2 + .../rbd-trash-list.component.spec.ts | 2 + .../rbd-trash-move-modal.component.spec.ts | 8 +- .../cephfs-clients.component.spec.ts | 8 +- .../cephfs-directories.component.spec.ts | 2 +- .../cephfs-directories.component.ts | 4 +- .../cephfs-list/cephfs-list.component.html | 4 +- .../cephfs-list/cephfs-list.component.spec.ts | 3 +- .../cephfs-list/cephfs-list.component.ts | 7 +- .../cephfs-tabs/cephfs-tabs.component.html | 2 +- .../cephfs-tabs/cephfs-tabs.component.spec.ts | 37 +- .../cephfs-tabs/cephfs-tabs.component.ts | 11 +- .../src/app/ceph/cluster/cluster.module.ts | 4 +- .../configuration-details.component.html | 36 +- .../configuration-details.component.ts | 10 +- .../configuration-form.component.ts | 10 +- .../configuration.component.html | 4 +- .../configuration.component.spec.ts | 2 + .../configuration/configuration.component.ts | 4 +- .../cluster/crushmap/crushmap.component.ts | 4 +- .../host-details/host-details.component.html | 4 +- .../host-details.component.spec.ts | 7 +- .../host-details/host-details.component.ts | 5 +- .../ceph/cluster/hosts/hosts.component.html | 9 +- .../cluster/hosts/hosts.component.spec.ts | 5 +- .../app/ceph/cluster/hosts/hosts.component.ts | 12 +- .../inventory-devices.component.spec.ts | 9 +- .../inventory/inventory.component.spec.ts | 2 + .../app/ceph/cluster/logs/logs.component.scss | 11 +- .../mgr-module-details.component.html | 2 +- .../mgr-module-details.component.spec.ts | 3 +- .../mgr-module-details.component.ts | 8 +- .../mgr-module-list.component.html | 4 +- .../mgr-module-list.component.spec.ts | 2 + .../mgr-module-list.component.ts | 4 +- .../ceph/cluster/monitor/monitor.component.ts | 6 +- .../osd-details/osd-details.component.html | 2 +- .../osd-details/osd-details.component.spec.ts | 3 +- .../osd/osd-details/osd-details.component.ts | 7 +- ...devices-selection-groups.component.spec.ts | 9 +- ...-devices-selection-modal.component.spec.ts | 2 + .../osd-flags-modal.component.spec.ts | 2 +- .../osd/osd-form/osd-form.component.spec.ts | 2 + .../osd/osd-list/osd-list.component.html | 7 +- .../osd/osd-list/osd-list.component.spec.ts | 4 +- .../osd/osd-list/osd-list.component.ts | 4 +- .../active-alert-list.component.html | 25 +- .../active-alert-list.component.spec.ts | 2 + .../active-alert-list.component.ts | 4 +- .../rules-list/rules-list.component.html | 7 +- .../rules-list/rules-list.component.spec.ts | 3 +- .../rules-list/rules-list.component.ts | 14 +- .../silence-list/silence-list.component.html | 24 +- .../silence-list.component.spec.ts | 2 + .../silence-list/silence-list.component.ts | 4 +- .../service-daemon-list.component.html | 8 + .../service-daemon-list.component.ts | 25 +- .../service-details.component.html | 4 +- .../cluster/services/services.component.html | 5 +- .../services/services.component.spec.ts | 16 +- .../cluster/services/services.component.ts | 8 +- .../telemetry/telemetry.component.html | 310 + .../telemetry/telemetry.component.scss | 0 .../telemetry/telemetry.component.spec.ts | 180 + .../cluster/telemetry/telemetry.component.ts | 210 + .../health-pie/health-pie.component.ts | 4 +- .../dashboard/health/health.component.scss | 21 +- .../info-card/info-card.component.scss | 7 +- .../nfs-details/nfs-details.component.html | 2 +- .../nfs-details/nfs-details.component.spec.ts | 56 +- .../nfs/nfs-details/nfs-details.component.ts | 8 +- .../ceph/nfs/nfs-list/nfs-list.component.html | 4 +- .../nfs/nfs-list/nfs-list.component.spec.ts | 2 + .../ceph/nfs/nfs-list/nfs-list.component.ts | 4 +- .../performance-counter.component.spec.ts | 3 +- .../crush-rule-form-modal.component.html | 18 +- .../crush-rule-form-modal.component.spec.ts | 4 +- .../crush-rule-form-modal.component.ts | 107 +- ...ure-code-profile-form-modal.component.html | 62 +- ...-code-profile-form-modal.component.spec.ts | 294 +- ...asure-code-profile-form-modal.component.ts | 267 +- .../pool-details/pool-details.component.html | 10 +- .../pool-details.component.spec.ts | 21 +- .../pool-details/pool-details.component.ts | 7 +- .../pool/pool-form/pool-form.component.html | 30 +- .../pool-form/pool-form.component.spec.ts | 188 +- .../pool/pool-form/pool-form.component.ts | 205 +- .../pool/pool-list/pool-list.component.html | 6 +- .../pool-list/pool-list.component.spec.ts | 24 +- .../pool/pool-list/pool-list.component.ts | 22 +- .../rgw-bucket-details.component.html | 56 +- .../rgw-bucket-details.component.ts | 16 +- .../rgw-bucket-form.component.spec.ts | 2 +- .../rgw-bucket-form.component.ts | 2 +- .../rgw-bucket-list.component.html | 4 +- .../rgw-bucket-list.component.spec.ts | 2 + .../rgw-bucket-list.component.ts | 4 +- .../rgw-daemon-details.component.html | 4 +- .../rgw-daemon-details.component.spec.ts | 3 +- .../rgw-daemon-details.component.ts | 7 +- .../rgw-daemon-list.component.html | 15 +- .../rgw-daemon-list.component.spec.ts | 42 +- .../rgw-daemon-list.component.ts | 27 +- .../rgw-user-details.component.html | 2 +- .../rgw-user-details.component.spec.ts | 19 +- .../rgw-user-details.component.ts | 6 +- .../rgw-user-list.component.html | 4 +- .../rgw-user-list.component.spec.ts | 9 +- .../rgw-user-list/rgw-user-list.component.ts | 4 +- .../smart-list/smart-list.component.spec.ts | 3 +- .../shared/smart-list/smart-list.component.ts | 7 +- .../app/core/auth/login/login.component.html | 55 +- .../app/core/auth/login/login.component.scss | 26 +- .../role-details/role-details.component.html | 2 +- .../role-details.component.spec.ts | 39 +- .../role-details/role-details.component.ts | 7 +- .../auth/role-list/role-list.component.html | 4 +- .../role-list/role-list.component.spec.ts | 2 + .../auth/role-list/role-list.component.ts | 4 +- .../auth/user-form/user-form.component.html | 3 + .../auth/user-form/user-form.component.ts | 6 +- .../user-list/user-list.component.spec.ts | 2 + .../user-password-form.component.spec.ts | 2 +- .../user-password-form.component.ts | 6 +- .../login-layout/login-layout.component.html | 12 +- .../login-layout/login-layout.component.scss | 4 +- .../login-layout.component.spec.ts | 2 + .../navigation/about/about.component.html | 95 +- .../navigation/about/about.component.scss | 35 +- .../administration.component.html | 5 + .../administration.component.ts | 5 +- .../app/core/navigation/navigation.module.ts | 2 - .../navigation/navigation.component.html | 7 +- .../navigation/navigation.component.scss | 4 + .../navigation/navigation.component.spec.ts | 4 +- .../navigation/navigation.component.ts | 4 +- .../src/app/shared/api/auth.service.spec.ts | 2 +- .../src/app/shared/api/auth.service.ts | 2 +- .../app/shared/api/rgw-site.service.spec.ts | 12 +- .../src/app/shared/api/rgw-site.service.ts | 6 +- .../app/shared/api/telemetry.service.spec.ts | 58 + .../src/app/shared/api/telemetry.service.ts | 25 + .../src/app/shared/api/user.service.ts | 12 + .../crush.node.selection.class.spec.ts | 207 + .../classes/crush.node.selection.class.ts | 140 + .../shared/classes/list-with-details.class.ts | 7 + .../language-selector.component.spec.ts | 180 +- .../language-selector.component.ts | 118 +- .../select/select.component.spec.ts | 2 +- .../table-actions.component.scss | 6 - .../datatable/table/table.component.html | 56 +- .../datatable/table/table.component.scss | 64 +- .../datatable/table/table.component.spec.ts | 77 + .../shared/datatable/table/table.component.ts | 83 +- .../src/app/shared/decorators/cd-encode.ts | 2 +- .../copy2clipboard-button.directive.spec.ts | 12 +- .../copy2clipboard-button.directive.ts | 10 +- .../models/cd-table-column-filters-change.ts | 2 +- .../src/app/shared/models/pool-form-info.ts | 1 + .../shared/pipes/relative-date.pipe.spec.ts | 8 +- .../services/api-interceptor.service.ts | 2 +- .../services/password-policy.service.ts | 10 +- .../services/text-to-download.service.spec.ts | 20 + .../services/text-to-download.service.ts | 14 + ...h_Logo_Standard_RGB_Reversed_120411_fa.png | Bin 0 -> 28175 bytes .../frontend/src/assets/ceph_background.gif | Bin 0 -> 98115 bytes .../mgr/dashboard/frontend/src/styles.scss | 1 + .../src/styles/bootstrap-extends.scss | 89 + .../frontend/src/styles/defaults.scss | 7 +- .../frontend/src/styles/vendor.variables.scss | 2 +- .../mgr/dashboard/frontend/tsconfig.json | 4 +- .../mgr/dashboard/run-frontend-e2e-tests.sh | 138 +- .../mgr/dashboard/run-frontend-unittests.sh | 15 +- .../mgr/dashboard/services/ceph_service.py | 21 +- .../mgr/dashboard/services/iscsi_cli.py | 8 +- .../mgr/dashboard/services/iscsi_config.py | 25 +- ceph/src/pybind/mgr/dashboard/services/rbd.py | 14 + .../mgr/dashboard/services/rgw_client.py | 10 + .../mgr/dashboard/tests/test_ceph_service.py | 65 + .../pybind/mgr/dashboard/tests/test_home.py | 2 +- .../mgr/dashboard/tests/test_rbd_service.py | 29 +- .../mgr/dashboard/tests/test_rgw_client.py | 21 +- ceph/src/pybind/mgr/devicehealth/module.py | 4 +- ceph/src/pybind/mgr/insights/module.py | 2 +- ceph/src/pybind/mgr/mgr_module.py | 16 +- .../src/pybind/mgr/orchestrator/_interface.py | 111 +- ceph/src/pybind/mgr/orchestrator/module.py | 505 +- ceph/src/pybind/mgr/progress/module.py | 12 +- ceph/src/pybind/mgr/prometheus/module.py | 32 +- .../mgr/rbd_support/trash_purge_schedule.py | 2 +- ceph/src/pybind/mgr/requirements.txt | 5 +- ceph/src/pybind/mgr/restful/api/config.py | 2 +- ceph/src/pybind/mgr/restful/module.py | 4 +- ceph/src/pybind/mgr/rook/module.py | 10 +- ceph/src/pybind/mgr/rook/rook_cluster.py | 4 +- ceph/src/pybind/mgr/status/module.py | 8 +- ceph/src/pybind/mgr/telemetry/module.py | 39 +- .../pybind/mgr/test_orchestrator/module.py | 18 +- ceph/src/pybind/mgr/tests/__init__.py | 57 +- .../src/pybind/mgr/volumes/fs/async_cloner.py | 6 +- .../fs/operations/versions/subvolume_base.py | 12 +- .../fs/operations/versions/subvolume_v1.py | 34 +- ceph/src/pybind/mgr/volumes/fs/volume.py | 35 +- ceph/src/pybind/mgr/volumes/module.py | 25 +- ceph/src/pybind/rados/rados.pyx | 4 +- ceph/src/pybind/rbd/rbd.pyx | 2 +- .../ceph/deployment/drive_group.py | 3 +- .../deployment/drive_selection/selector.py | 6 +- .../ceph/deployment/inventory.py | 1 + .../ceph/deployment/service_spec.py | 105 +- .../ceph/tests/test_drive_group.py | 4 +- ceph/src/python-common/ceph/tests/utils.py | 6 + ceph/src/rbd_fuse/rbd-fuse.cc | 3 - ceph/src/rgw/CMakeLists.txt | 3 + ceph/src/rgw/rgw-orphan-list | 90 + ceph/src/rgw/rgw_admin.cc | 73 +- ceph/src/rgw/rgw_amqp.cc | 109 +- ceph/src/rgw/rgw_amqp.h | 2 +- ceph/src/rgw/rgw_auth.h | 1 + ceph/src/rgw/rgw_bucket.cc | 8 +- ceph/src/rgw/rgw_cors.cc | 11 +- ceph/src/rgw/rgw_lc.cc | 714 +- ceph/src/rgw/rgw_lc.h | 45 +- ceph/src/rgw/rgw_obj_manifest.h | 7 +- ceph/src/rgw/rgw_op.cc | 32 +- ceph/src/rgw/rgw_op.h | 7 + ceph/src/rgw/rgw_orphan.cc | 624 ++ ceph/src/rgw/rgw_orphan.h | 77 + ceph/src/rgw/rgw_perf_counters.cc | 1 + ceph/src/rgw/rgw_perf_counters.h | 1 + ceph/src/rgw/rgw_pubsub_push.cc | 60 +- ceph/src/rgw/rgw_rados.cc | 17 +- ceph/src/rgw/rgw_swift_auth.h | 24 +- ceph/src/rgw/rgw_url.cc | 13 +- ceph/src/rgw/services/svc_tier_rados.h | 10 +- ceph/src/rgw/services/svc_user_rados.cc | 10 +- ceph/src/rocksdb/utilities/env_librados.cc | 4 +- ceph/src/test/cli/radosgw-admin/help.t | 8 +- ceph/src/test/cls_queue/test_cls_queue.cc | 44 + ceph/src/test/fs/CMakeLists.txt | 6 +- ceph/src/test/fs/test_ino_release_cb.cc | 77 + ceph/src/test/librbd/test_mock_Watcher.cc | 12 - ceph/src/test/objectstore/Allocator_bench.cc | 34 +- ceph/src/test/objectstore/Allocator_test.cc | 38 +- ceph/src/test/objectstore/CMakeLists.txt | 18 + .../objectstore/fastbmap_allocator_test.cc | 128 + .../test/objectstore/hybrid_allocator_test.cc | 231 + ceph/src/test/objectstore/test_bdev.cc | 111 + ceph/src/test/objectstore/test_bluefs.cc | 9 +- .../test/objectstore/test_bluestore_types.cc | 2 + ceph/src/test/pybind/test_rbd.py | 2 +- .../journal/test_mock_Replayer.cc | 73 +- ceph/src/test/rgw/CMakeLists.txt | 3 + ceph/src/test/rgw/rgw_multi/multisite.py | 3 +- ceph/src/test/rgw/rgw_multi/tests.py | 25 +- ceph/src/test/rgw/rgw_multi/tests_az.py | 18 +- ceph/src/test/rgw/rgw_multi/tests_es.py | 15 +- ceph/src/test/rgw/rgw_multi/tests_ps.py | 149 +- ceph/src/test/rgw/rgw_multi/zone_cloud.py | 6 +- ceph/src/test/rgw/rgw_multi/zone_es.py | 2 +- ceph/src/test/rgw/rgw_multi/zone_ps.py | 34 +- ceph/src/test/rgw/test-ceph-diff-sorted.sh | 108 + ceph/src/test/rgw/test_multi.py | 4 +- ceph/src/test/rgw/test_rgw_amqp.cc | 48 +- ceph/src/test/rgw/test_rgw_url.cc | 11 +- ceph/src/tools/CMakeLists.txt | 4 + ceph/src/tools/ceph-diff-sorted.cc | 173 + ceph/src/tools/cephfs/cephfs-shell | 61 +- ceph/src/tools/cephfs/tox.ini | 3 +- ceph/src/tools/rbd_mirror/ImageReplayer.cc | 30 +- ceph/src/tools/rbd_mirror/ImageReplayer.h | 14 +- ceph/src/tools/rbd_mirror/InstanceReplayer.cc | 12 +- .../image_replayer/journal/Replayer.cc | 163 +- .../image_replayer/journal/Replayer.h | 12 +- ceph/src/vstart.sh | 9 - 740 files changed, 21561 insertions(+), 13831 deletions(-) create mode 100644 ceph/doc/images/dashboard/invalid-credentials.png create mode 100644 ceph/doc/man/8/ceph-diff-sorted.rst create mode 100644 ceph/doc/man/8/rgw-orphan-list.rst create mode 100644 ceph/doc/radosgw/orphans.rst create mode 100644 ceph/monitoring/grafana/dashboards/radosgw-sync-overview.json create mode 100644 ceph/monitoring/grafana/dashboards/rbd-details.json create mode 100644 ceph/qa/.teuthology_branch rename ceph/qa/objectstore/{bluestore-avl.yaml => bluestore-hybrid.yaml} (94%) mode change 120000 => 100644 ceph/qa/suites/rados/cephadm/upgrade/fixed-2.yaml delete mode 120000 ceph/qa/suites/rados/cephadm/with-work/distro/ubuntu_latest.yaml delete mode 120000 ceph/qa/suites/rados/cephadm/workunits/distro/ubuntu_latest.yaml create mode 100644 ceph/qa/suites/rgw/tools/+ create mode 120000 ceph/qa/suites/rgw/tools/.qa create mode 120000 ceph/qa/suites/rgw/tools/centos_latest.yaml create mode 100644 ceph/qa/suites/rgw/tools/cluster.yaml create mode 100644 ceph/qa/suites/rgw/tools/tasks.yaml create mode 100644 ceph/qa/tasks/mgr/dashboard/test_telemetry.py create mode 100644 ceph/qa/tasks/tests/test_cephadm.py create mode 100755 ceph/qa/workunits/rgw/test_rgw_orphan_list.sh rename ceph/src/include/cephfs/{ceph_statx.h => ceph_ll_client.h} (61%) create mode 100644 ceph/src/os/bluestore/HybridAllocator.cc create mode 100644 ceph/src/os/bluestore/HybridAllocator.h create mode 100644 ceph/src/pybind/mgr/cephadm/inventory.py delete mode 100644 ceph/src/pybind/mgr/cephadm/osd.py create mode 100644 ceph/src/pybind/mgr/cephadm/schedule.py create mode 100644 ceph/src/pybind/mgr/cephadm/services/__init__.py create mode 100644 ceph/src/pybind/mgr/cephadm/services/cephadmservice.py create mode 100644 ceph/src/pybind/mgr/cephadm/services/iscsi.py create mode 100644 ceph/src/pybind/mgr/cephadm/services/monitoring.py rename ceph/src/pybind/mgr/cephadm/{ => services}/nfs.py (57%) create mode 100644 ceph/src/pybind/mgr/cephadm/services/osd.py create mode 100644 ceph/src/pybind/mgr/cephadm/template.py create mode 100644 ceph/src/pybind/mgr/cephadm/templates/services/alertmanager/alertmanager.yml.j2 create mode 100644 ceph/src/pybind/mgr/cephadm/templates/services/grafana/ceph-dashboard.yml.j2 create mode 100644 ceph/src/pybind/mgr/cephadm/templates/services/grafana/grafana.ini.j2 create mode 100644 ceph/src/pybind/mgr/cephadm/templates/services/iscsi/iscsi-gateway.cfg.j2 create mode 100644 ceph/src/pybind/mgr/cephadm/templates/services/nfs/ganesha.conf.j2 create mode 100644 ceph/src/pybind/mgr/cephadm/templates/services/prometheus/prometheus.yml.j2 create mode 100644 ceph/src/pybind/mgr/cephadm/tests/test_services.py create mode 100644 ceph/src/pybind/mgr/cephadm/tests/test_template.py create mode 100644 ceph/src/pybind/mgr/cephadm/upgrade.py delete mode 100644 ceph/src/pybind/mgr/dashboard/conftest.py create mode 100644 ceph/src/pybind/mgr/dashboard/controllers/telemetry.py create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress.json create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/block/images.e2e-spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/block/images.po.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/block/iscsi.e2e-spec.ts rename ceph/src/pybind/mgr/dashboard/frontend/{e2e => cypress/integration}/block/iscsi.po.ts (56%) create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/block/mirroring.e2e-spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/block/mirroring.po.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/configuration.e2e-spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/configuration.po.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/crush-map.e2e-spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/crush-map.po.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/hosts.e2e-spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/hosts.po.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/logs.e2e-spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/logs.po.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/mgr-modules.e2e-spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/mgr-modules.po.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/monitors.e2e-spec.ts rename ceph/src/pybind/mgr/dashboard/frontend/{e2e => cypress/integration}/cluster/monitors.po.ts (60%) create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/osds.e2e-spec.ts rename ceph/src/pybind/mgr/dashboard/frontend/{e2e => cypress/integration}/cluster/osds.po.ts (63%) create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/filesystems/filesystems.e2e-spec.ts rename ceph/src/pybind/mgr/dashboard/frontend/{e2e => cypress/integration}/filesystems/filesystems.po.ts (62%) create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/nfs/nfs.e2e-spec.ts rename ceph/src/pybind/mgr/dashboard/frontend/{e2e => cypress/integration}/nfs/nfs.po.ts (63%) create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/pools/pools.e2e-spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/pools/pools.po.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/rgw/buckets.e2e-spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/rgw/buckets.po.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/rgw/daemons.e2e-spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/rgw/daemons.po.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/rgw/users.e2e-spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/rgw/users.po.ts rename ceph/src/pybind/mgr/dashboard/frontend/{e2e => cypress/integration}/ui/dashboard.e2e-spec.ts (55%) create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/ui/dashboard.po.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/ui/notification.e2e-spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/ui/notification.po.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/ui/role-mgmt.e2e-spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/ui/role-mgmt.po.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/ui/user-mgmt.e2e-spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/ui/user-mgmt.po.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/support/commands.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/support/index.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/cypress/tsconfig.json create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/2.5b0a7ba1cfec198ba8ed.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/2.d36650ee0a92dfd05faa.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/6.4299ba94dd69b0b04046.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/6.c0ad55977c0a220b3203.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/7.904941689cb18cba0036.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/7.cf5f8c70f123c771366b.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/8.988fe96f3a1b2f8c64cd.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/8.f45b72794d78f44d11b9.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/9.a6e2f7400e0a62470111.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/9.d895914fbc785284c5be.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/assets/Ceph_Logo_Standard_RGB_Reversed_120411_fa.png create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/assets/ceph_background.gif create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/ceph_background.7c127240f022e7baf57d.gif create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/main.a10546ecbf9c49d65e18.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/main.bbd2d62de0d7a670cae3.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/runtime.2fd4e31a298ae797b302.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/runtime.992511a1840a2561365a.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/styles.4f0058c288677d311127.css delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/styles.f15465861b49727cf28a.css delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/block/images.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/block/images.po.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/block/iscsi.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/block/mirroring.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/block/mirroring.po.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/cluster/configuration.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/cluster/configuration.po.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/cluster/crush-map.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/cluster/crush-map.po.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/cluster/hosts.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/cluster/hosts.po.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/cluster/logs.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/cluster/logs.po.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/cluster/mgr-modules.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/cluster/mgr-modules.po.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/cluster/monitors.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/cluster/osds.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/filesystems/filesystems.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/nfs/nfs.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/page-helper.po.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/pools/pools.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/pools/pools.po.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/rgw/buckets.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/rgw/buckets.po.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/rgw/daemons.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/rgw/daemons.po.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/rgw/users.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/rgw/users.po.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/tsconfig.e2e.json delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/ui/dashboard.po.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/ui/notification.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/ui/notification.po.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/ui/role-mgmt.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/ui/role-mgmt.po.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/ui/user-mgmt.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/ui/user-mgmt.po.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/protractor.conf.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/telemetry/telemetry.component.html create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/telemetry/telemetry.component.scss create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/telemetry/telemetry.component.spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/telemetry/telemetry.component.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/api/telemetry.service.spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/api/telemetry.service.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/classes/crush.node.selection.class.spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/classes/crush.node.selection.class.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/classes/list-with-details.class.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/services/text-to-download.service.spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/services/text-to-download.service.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/assets/Ceph_Logo_Standard_RGB_Reversed_120411_fa.png create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/assets/ceph_background.gif create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/styles/bootstrap-extends.scss create mode 100644 ceph/src/pybind/mgr/dashboard/tests/test_ceph_service.py create mode 100755 ceph/src/rgw/rgw-orphan-list create mode 100644 ceph/src/test/fs/test_ino_release_cb.cc create mode 100755 ceph/src/test/objectstore/hybrid_allocator_test.cc create mode 100755 ceph/src/test/objectstore/test_bdev.cc create mode 100755 ceph/src/test/rgw/test-ceph-diff-sorted.sh create mode 100644 ceph/src/tools/ceph-diff-sorted.cc diff --git a/ceph/CMakeLists.txt b/ceph/CMakeLists.txt index e226b8dff..bddd18497 100644 --- a/ceph/CMakeLists.txt +++ b/ceph/CMakeLists.txt @@ -667,4 +667,4 @@ add_custom_target(tags DEPENDS ctags) find_package(CppCheck) find_package(IWYU) -set(VERSION 15.2.3) +set(VERSION 15.2.4) diff --git a/ceph/PendingReleaseNotes b/ceph/PendingReleaseNotes index 6e07ce6d4..547a3fe88 100644 --- a/ceph/PendingReleaseNotes +++ b/ceph/PendingReleaseNotes @@ -1,320 +1,53 @@ ->=15.0.0 +>=15.2.4 -------- -* CVE-2020-10736: Fixes an authorization bypass in monitor and manager daemons - -* The RGW "num_rados_handles" has been removed. - * If you were using a value of "num_rados_handles" greater than 1 - multiply your current "objecter_inflight_ops" and - "objecter_inflight_op_bytes" paramaeters by the old - "num_rados_handles" to get the same throttle behavior. - -* Ceph now packages python bindings for python3.6 instead of - python3.4, because python3 in EL7/EL8 is now using python3.6 - as the native python3. see the `announcement _` - for more details on the background of this change. - -* librbd now uses a write-around cache policy be default, - replacing the previous write-back cache policy default. - This cache policy allows librbd to immediately complete - write IOs while they are still in-flight to the OSDs. - Subsequent flush requests will ensure all in-flight - write IOs are completed prior to completing. The - librbd cache policy can be controlled via a new - "rbd_cache_policy" configuration option. - -* librbd now includes a simple IO scheduler which attempts to - batch together multiple IOs against the same backing RBD - data block object. The librbd IO scheduler policy can be - controlled via a new "rbd_io_scheduler" configuration - option. - -* RGW: radosgw-admin introduces two subcommands that allow the - managing of expire-stale objects that might be left behind after a - bucket reshard in earlier versions of RGW. One subcommand lists such - objects and the other deletes them. Read the troubleshooting section - of the dynamic resharding docs for details. - -* RGW: Bucket naming restrictions have changed and likely to cause - InvalidBucketName errors. We recommend to set ``rgw_relaxed_s3_bucket_names`` - option to true as a workaround. - -* In the Zabbix Mgr Module there was a typo in the key being send - to Zabbix for PGs in backfill_wait state. The key that was sent - was 'wait_backfill' and the correct name is 'backfill_wait'. - Update your Zabbix template accordingly so that it accepts the - new key being send to Zabbix. - -* zabbix plugin for ceph manager now includes osd and pool - discovery. Update of zabbix_template.xml is needed - to receive per-pool (read/write throughput, diskspace usage) - and per-osd (latency, status, pgs) statistics - -* The format of all date + time stamps has been modified to fully - conform to ISO 8601. The old format (``YYYY-MM-DD - HH:MM:SS.ssssss``) excluded the ``T`` separator between the date and - time and was rendered using the local time zone without any explicit - indication. The new format includes the separator as well as a - ``+nnnn`` or ``-nnnn`` suffix to indicate the time zone, or a ``Z`` - suffix if the time is UTC. For example, - ``2019-04-26T18:40:06.225953+0100``. - - Any code or scripts that was previously parsing date and/or time - values from the JSON or XML structure CLI output should be checked - to ensure it can handle ISO 8601 conformant values. Any code - parsing date or time values from the unstructured human-readable - output should be modified to parse the structured output instead, as - the human-readable output may change without notice. - -* The ``bluestore_no_per_pool_stats_tolerance`` config option has been - replaced with ``bluestore_fsck_error_on_no_per_pool_stats`` - (default: false). The overall default behavior has not changed: - fsck will warn but not fail on legacy stores, and repair will - convert to per-pool stats. - -* The disaster-recovery related 'ceph mon sync force' command has been - replaced with 'ceph daemon <...> sync_force'. - -* The ``osd_recovery_max_active`` option now has - ``osd_recovery_max_active_hdd`` and ``osd_recovery_max_active_ssd`` - variants, each with different default values for HDD and SSD-backed - OSDs, respectively. By default ``osd_recovery_max_active`` now - defaults to zero, which means that the OSD will conditionally use - the HDD or SSD option values. Administrators who have customized - this value may want to consider whether they have set this to a - value similar to the new defaults (3 for HDDs and 10 for SSDs) and, - if so, remove the option from their configuration entirely. - -* monitors now have a `ceph osd info` command that will provide information - on all osds, or provided osds, thus simplifying the process of having to - parse `osd dump` for the same information. - -* The structured output of ``ceph status`` or ``ceph -s`` is now more - concise, particularly the `mgrmap` and `monmap` sections, and the - structure of the `osdmap` section has been cleaned up. - -* A health warning is now generated if the average osd heartbeat ping - time exceeds a configurable threshold for any of the intervals - computed. The OSD computes 1 minute, 5 minute and 15 minute - intervals with average, minimum and maximum values. New configuration - option ``mon_warn_on_slow_ping_ratio`` specifies a percentage of - ``osd_heartbeat_grace`` to determine the threshold. A value of zero - disables the warning. New configuration option - ``mon_warn_on_slow_ping_time`` specified in milliseconds over-rides the - computed value, causes a warning - when OSD heartbeat pings take longer than the specified amount. - New admin command ``ceph daemon mgr.# dump_osd_network [threshold]`` command will - list all connections with a ping time longer than the specified threshold or - value determined by the config options, for the average for any of the 3 intervals. - New admin command ``ceph daemon osd.# dump_osd_network [threshold]`` will - do the same but only including heartbeats initiated by the specified OSD. - -* Inline data support for CephFS has been deprecated. When setting the flag, - users will see a warning to that effect, and enabling it now requires the - ``--yes-i-really-really-mean-it`` flag. If the MDS is started on a - filesystem that has it enabled, a health warning is generated. Support for - this feature will be removed in a future release. - -* ``ceph {set,unset} full`` is not supported anymore. We have been using - ``full`` and ``nearfull`` flags in OSD map for tracking the fullness status - of a cluster back since the Hammer release, if the OSD map is marked ``full`` - all write operations will be blocked until this flag is removed. In the - Infernalis release and Linux kernel 4.7 client, we introduced the per-pool - full/nearfull flags to track the status for a finer-grained control, so the - clients will hold the write operations if either the cluster-wide ``full`` - flag or the per-pool ``full`` flag is set. This was a compromise, as we - needed to support the cluster with and without per-pool ``full`` flags - support. But this practically defeated the purpose of introducing the - per-pool flags. So, in the Mimic release, the new flags finally took the - place of their cluster-wide counterparts, as the monitor started removing - these two flags from OSD map. So the clients of Infernalis and up can benefit - from this change, as they won't be blocked by the full pools which they are - not writing to. In this release, ``ceph {set,unset} full`` is now considered - as an invalid command. And the clients will continue honoring both the - cluster-wide and per-pool flags to be backward comaptible with pre-infernalis - clusters. - -* The telemetry module now reports more information. - - First, there is a new 'device' channel, enabled by default, that - will report anonymized hard disk and SSD health metrics to - telemetry.ceph.com in order to build and improve device failure - prediction algorithms. If you are not comfortable sharing device - metrics, you can disable that channel first before re-opting-in:: - - ceph config set mgr mgr/telemetry/channel_device false - - Second, we now report more information about CephFS file systems, - including: - - - how many MDS daemons (in total and per file system) - - which features are (or have been) enabled - - how many data pools - - approximate file system age (year + month of creation) - - how many files, bytes, and snapshots - - how much metadata is being cached - - We have also added: - - - which Ceph release the monitors are running - - whether msgr v1 or v2 addresses are used for the monitors - - whether IPv4 or IPv6 addresses are used for the monitors - - whether RADOS cache tiering is enabled (and which mode) - - whether pools are replicated or erasure coded, and - which erasure code profile plugin and parameters are in use - - how many hosts are in the cluster, and how many hosts have each type of daemon - - whether a separate OSD cluster network is being used - - how many RBD pools and images are in the cluster, and how many pools have RBD mirroring enabled - - how many RGW daemons, zones, and zonegroups are present; which RGW frontends are in use - - aggregate stats about the CRUSH map, like which algorithms are used, how - big buckets are, how many rules are defined, and what tunables are in - use - - If you had telemetry enabled, you will need to re-opt-in with:: - - ceph telemetry on - - You can view exactly what information will be reported first with:: - - ceph telemetry show # see everything - ceph telemetry show basic # basic cluster info (including all of the new info) - -* Following invalid settings now are not tolerated anymore - for the command `ceph osd erasure-code-profile set xxx`. - * invalid `m` for "reed_sol_r6_op" erasure technique - * invalid `m` and invalid `w` for "liber8tion" erasure technique - -* New OSD daemon command dump_recovery_reservations which reveals the - recovery locks held (in_progress) and waiting in priority queues. - -* New OSD daemon command dump_scrub_reservations which reveals the - scrub reservations that are held for local (primary) and remote (replica) PGs. - -* Previously, ``ceph tell mgr ...`` could be used to call commands - implemented by mgr modules. This is no longer supported. Since - luminous, using ``tell`` has not been necessary: those same commands - are also accessible without the ``tell mgr`` portion (e.g., ``ceph - tell mgr influx foo`` is the same as ``ceph influx foo``. ``ceph - tell mgr ...`` will now call admin commands--the same set of - commands accessible via ``ceph daemon ...`` when you are logged into - the appropriate host. - -* The ``ceph tell`` and ``ceph daemon`` commands have been unified, - such that all such commands are accessible via either interface. - Note that ceph-mgr tell commands are accessible via either ``ceph - tell mgr ...`` or ``ceph tell mgr. ...``, and it is only - possible to send tell commands to the active daemon (the standbys do - not accept incoming connections over the network). - -* Ceph will now issue a health warning if a RADOS pool as a ``pg_num`` - value that is not a power of two. This can be fixed by adjusting - the pool to a nearby power of two:: - - ceph osd pool set pg_num - - Alternatively, the warning can be silenced with:: - - ceph config set global mon_warn_on_pool_pg_num_not_power_of_two false - -* The format of MDSs in `ceph fs dump` has changed. - -* The ``mds_cache_size`` config option is completely removed. Since luminous, - the ``mds_cache_memory_limit`` config option has been preferred to configure - the MDS's cache limits. - -* The ``pg_autoscale_mode`` is now set to ``on`` by default for newly - created pools, which means that Ceph will automatically manage the - number of PGs. To change this behavior, or to learn more about PG - autoscaling, see :ref:`pg-autoscaler`. Note that existing pools in - upgraded clusters will still be set to ``warn`` by default. - -* The pool parameter ``target_size_ratio``, used by the pg autoscaler, - has changed meaning. It is now normalized across pools, rather than - specifying an absolute ratio. For details, see :ref:`pg-autoscaler`. - If you have set target size ratios on any pools, you may want to set - these pools to autoscale ``warn`` mode to avoid data movement during - the upgrade:: - - ceph osd pool set pg_autoscale_mode warn - -* The ``upmap_max_iterations`` config option of mgr/balancer has been - renamed to ``upmap_max_optimizations`` to better match its behaviour. - -* ``mClockClientQueue`` and ``mClockClassQueue`` OpQueue - implementations have been removed in favor of of a single - ``mClockScheduler`` implementation of a simpler OSD interface. - Accordingly, the ``osd_op_queue_mclock*`` family of config options - has been removed in favor of the ``osd_mclock_scheduler*`` family - of options. - -* The config subsystem now searches dot ('.') delineated prefixes for - options. That means for an entity like ``client.foo.bar``, it's - overall configuration will be a combination of the global options, - ``client``, ``client.foo``, and ``client.foo.bar``. Previously, - only global, ``client``, and ``client.foo.bar`` options would apply. - This change may affect the configuration for clients that include a - ``.`` in their name. - - Note that this only applies to configuration options in the - monitor's database--config file parsing is not affected. - -* RGW: bucket listing performance on sharded bucket indexes has been - notably improved by heuristically -- and significantly, in many - cases -- reducing the number of entries requested from each bucket - index shard. - -* MDS default cache memory limit is now 4GB. - -* The behaviour of the ``-o`` argument to the rados tool has been reverted to - its orignal behaviour of indicating an output file. This reverts it to a more - consisten behaviour when compared to other tools. Specifying obect size is now - accomplished by using an upper case O ``-O``. - -* In certain rare cases, OSDs would self-classify themselves as type - 'nvme' instead of 'hdd' or 'ssd'. This appears to be limited to - cases where BlueStore was deployed with older versions of ceph-disk, - or manually without ceph-volume and LVM. Going forward, the OSD - will limit itself to only 'hdd' and 'ssd' (or whatever device class the user - manually specifies). - -* RGW: a mismatch between the bucket notification documentation and the actual - message format was fixed. This means that any endpoints receiving bucket - notification, will now receive the same notifications inside an JSON array - named 'Records'. Note that this does not affect pulling bucket notification - from a subscription in a 'pubsub' zone, as these are already wrapped inside - that array. - -* The configuration value ``osd_calc_pg_upmaps_max_stddev`` used for upmap - balancing has been removed. Instead use the mgr balancer config - ``upmap_max_deviation`` which now is an integer number of PGs of deviation - from the target PGs per OSD. This can be set with a command like - ``ceph config set mgr mgr/balancer/upmap_max_deviation 2``. The default - ``upmap_max_deviation`` is 1. There are situations where crush rules - would not allow a pool to ever have completely balanced PGs. For example, if - crush requires 1 replica on each of 3 racks, but there are fewer OSDs in 1 of - the racks. In those cases, the configuration value can be increased. - -* MDS daemons can now be assigned to manage a particular file system via the - new ``mds_join_fs`` option. The monitors will try to use only MDS for a file - system with mds_join_fs equal to the file system name (strong affinity). - Monitors may also deliberately failover an active MDS to a standby when the - cluster is otherwise healthy if the standby has stronger affinity. - -* RGW Multisite: A new fine grained bucket-granularity policy configuration - system has been introduced and it supersedes the previous coarse zone sync - configuration (specifically the ``sync_from`` and ``sync_from_all`` fields - in the zonegroup configuration. New configuration should only be configured - after all relevant zones in the zonegroup have been upgraded. - -* RGW S3: Support has been added for BlockPublicAccess set of APIs at a bucket - level, currently blocking/ignoring public acls & policies are supported. - User/Account level APIs are planned to be added in the future - -* RGW: The default number of bucket index shards for new buckets was raised - from 1 to 11 to increase the amount of write throughput for small buckets - and delay the onset of dynamic resharding. This change only affects new - deployments/zones. To change this default value on existing deployments, - use 'radosgw-admin zonegroup modify --bucket-index-max-shards=11'. - If the zonegroup is part of a realm, the change must be committed with - 'radosgw-admin period update --commit' - otherwise the change will take - effect after radosgws are restarted. +* Cephadm: There were a lot of small usability improvements and bug fixes: + + * Grafana when deployed by Cephadm now binds to all network interfaces. + * ``cephadm check-host`` now prints all detected problems at once. + * Cephadm now calls ``ceph dashboard set-grafana-api-ssl-verify false`` + when generating an SSL certificate for Grafana. + * The Alertmanager is now correctly pointed to the Ceph Dashboard + * ``cephadm adopt`` now supports adopting an Alertmanager + * ``ceph orch ps`` now supports filtering by service name + * ``ceph orch host ls`` now marks hosts as offline, if they are not + accessible. + +* Cephadm can now deploy NFS Ganesha services. For example, to deploy NFS with + a service id of mynfs, that will use the RADOS pool nfs-ganesha and namespace + nfs-ns:: + + ceph orch apply nfs mynfs nfs-ganesha nfs-ns + +* Cephadm: ``ceph orch ls --export`` now returns all service specifications in + yaml representation that is consumable by ``ceph orch apply``. In addition, + the commands ``orch ps`` and ``orch ls`` now support ``--format yaml`` and + ``--format json-pretty``. + +* Cephadm: ``ceph orch apply osd`` supports a ``--preview`` flag that prints a preview of + the OSD specification before deploying OSDs. This makes it possible to + verify that the specification is correct, before applying it. + +* RGW: The ``radosgw-admin`` sub-commands dealing with orphans -- + ``radosgw-admin orphans find``, ``radosgw-admin orphans find``, + ``radosgw-admin orphans find`` -- have been deprecated. They have + not been actively maintained and they store intermediate results on + the cluster, which could fill a nearly-full cluster. They have been + replaced by a tool, currently considered experimental, + ``rgw-orphan-list``. + +* RBD: The name of the rbd pool object that is used to store + rbd trash purge schedule is changed from "rbd_trash_trash_purge_schedule" + to "rbd_trash_purge_schedule". Users that have already started using + ``rbd trash purge schedule`` functionality and have per pool or namespace + schedules configured should copy "rbd_trash_trash_purge_schedule" + object to "rbd_trash_purge_schedule" before the upgrade and remove + "rbd_trash_purge_schedule" using the following commands in every RBD + pool and namespace where a trash purge schedule was previously + configured:: + + rados -p [-N namespace] cp rbd_trash_trash_purge_schedule rbd_trash_purge_schedule + rados -p [-N namespace] rm rbd_trash_trash_purge_schedule + + or use any other convenient way to restore the schedule after the + upgrade. diff --git a/ceph/alpine/APKBUILD b/ceph/alpine/APKBUILD index 37e9144be..e62d15946 100644 --- a/ceph/alpine/APKBUILD +++ b/ceph/alpine/APKBUILD @@ -1,7 +1,7 @@ # Contributor: John Coyle # Maintainer: John Coyle pkgname=ceph -pkgver=15.2.3 +pkgver=15.2.4 pkgrel=0 pkgdesc="Ceph is a distributed object store and file system" pkgusers="ceph" @@ -63,7 +63,7 @@ makedepends=" xmlstarlet yasm " -source="ceph-15.2.3.tar.bz2" +source="ceph-15.2.4.tar.bz2" subpackages=" $pkgname-base $pkgname-common @@ -116,7 +116,7 @@ _sysconfdir=/etc _udevrulesdir=/etc/udev/rules.d _python_sitelib=/usr/lib/python2.7/site-packages -builddir=$srcdir/ceph-15.2.3 +builddir=$srcdir/ceph-15.2.4 build() { export CEPH_BUILD_VIRTUALENV=$builddir @@ -396,7 +396,7 @@ libcephfs_dev() { pkgdesc="Ceph distributed file system client library headers" depends="libcephfs librados-devel" - _pkg $_includedir/cephfs ceph_statx.h libcephfs.h + _pkg $_includedir/cephfs ceph_ll_client.h libcephfs.h _pkg $_libdir libcephfs.so } diff --git a/ceph/alpine/APKBUILD.in b/ceph/alpine/APKBUILD.in index 7a3b598b7..9348f9ccf 100644 --- a/ceph/alpine/APKBUILD.in +++ b/ceph/alpine/APKBUILD.in @@ -396,7 +396,7 @@ libcephfs_dev() { pkgdesc="Ceph distributed file system client library headers" depends="libcephfs librados-devel" - _pkg $_includedir/cephfs ceph_statx.h libcephfs.h + _pkg $_includedir/cephfs ceph_ll_client.h libcephfs.h _pkg $_libdir libcephfs.so } diff --git a/ceph/ceph.spec b/ceph/ceph.spec index 6bd376c79..74a5f95be 100644 --- a/ceph/ceph.spec +++ b/ceph/ceph.spec @@ -98,7 +98,7 @@ # main package definition ################################################################################# Name: ceph -Version: 15.2.3 +Version: 15.2.4 Release: 0%{?dist} %if 0%{?fedora} || 0%{?rhel} Epoch: 2 @@ -114,7 +114,7 @@ License: LGPL-2.1 and LGPL-3.0 and CC-BY-SA-3.0 and GPL-2.0 and BSL-1.0 and BSD- Group: System/Filesystems %endif URL: http://ceph.com/ -Source0: %{?_remote_tarball_prefix}ceph-15.2.3.tar.bz2 +Source0: %{?_remote_tarball_prefix}ceph-15.2.4.tar.bz2 %if 0%{?suse_version} # _insert_obs_source_lines_here ExclusiveArch: x86_64 aarch64 ppc64le s390x @@ -645,9 +645,11 @@ Requires: python%{python3_pkgversion}-remoto Requires: cephadm = %{_epoch_prefix}%{version}-%{release} %if 0%{?suse_version} Requires: openssh +Requires: python%{python3_pkgversion}-Jinja2 %endif %if 0%{?rhel} || 0%{?fedora} Requires: openssh-clients +Requires: python%{python3_pkgversion}-jinja2 %endif %description mgr-cephadm ceph-mgr-cephadm is a ceph-mgr module for orchestration functions using @@ -1117,7 +1119,7 @@ This package provides Ceph’s default alerts for Prometheus. # common ################################################################################# %prep -%autosetup -p1 -n ceph-15.2.3 +%autosetup -p1 -n ceph-15.2.4 %build # LTO can be enabled as soon as the following GCC bug is fixed: @@ -1486,6 +1488,7 @@ exit 0 %{_mandir}/man8/ceph-authtool.8* %{_mandir}/man8/ceph-conf.8* %{_mandir}/man8/ceph-dencoder.8* +%{_mandir}/man8/ceph-diff-sorted.8* %{_mandir}/man8/ceph-rbdnamer.8* %{_mandir}/man8/ceph-syn.8* %{_mandir}/man8/ceph-post-file.8* @@ -1498,6 +1501,7 @@ exit 0 %{_mandir}/man8/rbd-replay.8* %{_mandir}/man8/rbd-replay-many.8* %{_mandir}/man8/rbd-replay-prep.8* +%{_mandir}/man8/rgw-orphan-list.8* %dir %{_datadir}/ceph/ %{_datadir}/ceph/known_hosts_drop.ceph.com %{_datadir}/ceph/id_rsa_drop.ceph.com @@ -1915,10 +1919,12 @@ fi %{_mandir}/man8/rbd-nbd.8* %files radosgw +%{_bindir}/ceph-diff-sorted %{_bindir}/radosgw %{_bindir}/radosgw-token %{_bindir}/radosgw-es %{_bindir}/radosgw-object-expirer +%{_bindir}/rgw-orphan-list %{_libdir}/libradosgw.so* %{_mandir}/man8/radosgw.8* %dir %{_localstatedir}/lib/ceph/radosgw @@ -2167,7 +2173,7 @@ fi %files -n libcephfs-devel %dir %{_includedir}/cephfs %{_includedir}/cephfs/libcephfs.h -%{_includedir}/cephfs/ceph_statx.h +%{_includedir}/cephfs/ceph_ll_client.h %{_libdir}/libcephfs.so %files -n python%{python3_pkgversion}-cephfs diff --git a/ceph/ceph.spec.in b/ceph/ceph.spec.in index 2d20cd2bc..244c45170 100644 --- a/ceph/ceph.spec.in +++ b/ceph/ceph.spec.in @@ -645,9 +645,11 @@ Requires: python%{python3_pkgversion}-remoto Requires: cephadm = %{_epoch_prefix}%{version}-%{release} %if 0%{?suse_version} Requires: openssh +Requires: python%{python3_pkgversion}-Jinja2 %endif %if 0%{?rhel} || 0%{?fedora} Requires: openssh-clients +Requires: python%{python3_pkgversion}-jinja2 %endif %description mgr-cephadm ceph-mgr-cephadm is a ceph-mgr module for orchestration functions using @@ -1486,6 +1488,7 @@ exit 0 %{_mandir}/man8/ceph-authtool.8* %{_mandir}/man8/ceph-conf.8* %{_mandir}/man8/ceph-dencoder.8* +%{_mandir}/man8/ceph-diff-sorted.8* %{_mandir}/man8/ceph-rbdnamer.8* %{_mandir}/man8/ceph-syn.8* %{_mandir}/man8/ceph-post-file.8* @@ -1498,6 +1501,7 @@ exit 0 %{_mandir}/man8/rbd-replay.8* %{_mandir}/man8/rbd-replay-many.8* %{_mandir}/man8/rbd-replay-prep.8* +%{_mandir}/man8/rgw-orphan-list.8* %dir %{_datadir}/ceph/ %{_datadir}/ceph/known_hosts_drop.ceph.com %{_datadir}/ceph/id_rsa_drop.ceph.com @@ -1915,10 +1919,12 @@ fi %{_mandir}/man8/rbd-nbd.8* %files radosgw +%{_bindir}/ceph-diff-sorted %{_bindir}/radosgw %{_bindir}/radosgw-token %{_bindir}/radosgw-es %{_bindir}/radosgw-object-expirer +%{_bindir}/rgw-orphan-list %{_libdir}/libradosgw.so* %{_mandir}/man8/radosgw.8* %dir %{_localstatedir}/lib/ceph/radosgw @@ -2167,7 +2173,7 @@ fi %files -n libcephfs-devel %dir %{_includedir}/cephfs %{_includedir}/cephfs/libcephfs.h -%{_includedir}/cephfs/ceph_statx.h +%{_includedir}/cephfs/ceph_ll_client.h %{_libdir}/libcephfs.so %files -n python%{python3_pkgversion}-cephfs diff --git a/ceph/changelog.upstream b/ceph/changelog.upstream index 7f81d0a40..10af93187 100644 --- a/ceph/changelog.upstream +++ b/ceph/changelog.upstream @@ -1,7 +1,13 @@ -ceph (15.2.3-1bionic) bionic; urgency=medium +ceph (15.2.4-1bionic) bionic; urgency=medium - -- Jenkins Build Slave User Fri, 29 May 2020 16:37:54 +0000 + -- Jenkins Build Slave User Tue, 30 Jun 2020 16:42:44 +0000 + +ceph (15.2.4-1) stable; urgency=medium + + * New upstream release + + -- Ceph Release Team Tue, 30 Jun 2020 15:40:49 +0000 ceph (15.2.3-1) stable; urgency=medium diff --git a/ceph/cmake/modules/FindFUSE.cmake b/ceph/cmake/modules/FindFUSE.cmake index 1d0766f33..b55a2d36f 100644 --- a/ceph/cmake/modules/FindFUSE.cmake +++ b/ceph/cmake/modules/FindFUSE.cmake @@ -19,23 +19,46 @@ if(APPLE) list(APPEND fuse_suffixes osxfuse) endif() -find_path( - FUSE_INCLUDE_DIR - NAMES fuse_common.h fuse_lowlevel.h fuse.h - PATH_SUFFIXES ${fuse_suffixes}) - -find_library(FUSE_LIBRARIES - NAMES ${fuse_names} - PATHS /usr/local/lib64 /usr/local/lib) - -foreach(ver "MAJOR" "MINOR") - file(STRINGS "${FUSE_INCLUDE_DIR}/fuse_common.h" fuse_ver_${ver}_line - REGEX "^#define[\t ]+FUSE_${ver}_VERSION[\t ]+[0-9]+$") - string(REGEX REPLACE ".*#define[\t ]+FUSE_${ver}_VERSION[\t ]+([0-9]+)$" - "\\1" FUSE_VERSION_${ver} "${fuse_ver_${ver}_line}") -endforeach() +find_package(PkgConfig QUIET) +if(PKG_CONFIG_FOUND) + pkg_search_module(PKG_FUSE QUIET ${fuse_names}) + + string(REGEX REPLACE "([0-9]+)\.([0-9]+)\.([0-9]+)" + "\\1" FUSE_MAJOR_VERSION "${PKG_FUSE_VERSION}") + string(REGEX REPLACE "([0-9]+)\.([0-9]+)\.([0-9]+)" + "\\2" FUSE_MINOR_VERSION "${PKG_FUSE_VERSION}") + + find_path( + FUSE_INCLUDE_DIR + NAMES fuse_common.h fuse_lowlevel.h fuse.h + HINTS ${PKG_FUSE_INCLUDE_DIRS} + PATH_SUFFIXES ${fuse_suffixes} + NO_DEFAULT_PATH) + + find_library(FUSE_LIBRARIES + NAMES ${fuse_names} + HINTS ${PKG_FUSE_LIBDIR} + NO_DEFAULT_PATH) +else() + find_path( + FUSE_INCLUDE_DIR + NAMES fuse_common.h fuse_lowlevel.h fuse.h + PATH_SUFFIXES ${fuse_suffixes}) + + find_library(FUSE_LIBRARIES + NAMES ${fuse_names} + PATHS /usr/local/lib64 /usr/local/lib) + + foreach(ver "MAJOR" "MINOR") + file(STRINGS "${FUSE_INCLUDE_DIR}/fuse_common.h" fuse_ver_${ver}_line + REGEX "^#define[\t ]+FUSE_${ver}_VERSION[\t ]+[0-9]+$") + string(REGEX REPLACE ".*#define[\t ]+FUSE_${ver}_VERSION[\t ]+([0-9]+)$" + "\\1" FUSE_${ver}_VERSION "${fuse_ver_${ver}_line}") + endforeach() +endif() + set(FUSE_VERSION - "${FUSE_VERSION_MAJOR}.${FUSE_VERSION_MINOR}") + "${FUSE_MAJOR_VERSION}.${FUSE_MINOR_VERSION}") include(FindPackageHandleStandardArgs) find_package_handle_standard_args(FUSE diff --git a/ceph/debian/cephadm.postinst b/ceph/debian/cephadm.postinst index e84b9e1f6..53d503e1e 100644 --- a/ceph/debian/cephadm.postinst +++ b/ceph/debian/cephadm.postinst @@ -25,7 +25,7 @@ case "$1" in # 1. create user if not existing if ! getent passwd | grep -q "^cephadm:"; then echo -n "Adding system user cephadm.." - adduser --quiet --system --disabled-password --gecos 'Ceph-dameon user for cephadm' --shell /bin/bash cephadm 2>/dev/null || true + adduser --quiet --system --disabled-password --gecos 'cephadm user for mgr/cephadm' --shell /bin/bash cephadm 2>/dev/null || true echo "..done" fi diff --git a/ceph/debian/control b/ceph/debian/control index d98b16111..9abc2bd75 100644 --- a/ceph/debian/control +++ b/ceph/debian/control @@ -351,7 +351,8 @@ Depends: ceph-mgr (= ${binary:Version}), python3-six, ${misc:Depends}, ${python:Depends}, - openssh-client + openssh-client, + python3-jinja2 Description: cephadm orchestrator module for ceph-mgr Ceph is a massively scalable, open-source, distributed storage system that runs on commodity hardware and delivers object, diff --git a/ceph/debian/libcephfs-dev.install b/ceph/debian/libcephfs-dev.install index e2cf6e9f3..fbc1e4b95 100644 --- a/ceph/debian/libcephfs-dev.install +++ b/ceph/debian/libcephfs-dev.install @@ -1,3 +1,3 @@ -usr/include/cephfs/ceph_statx.h +usr/include/cephfs/ceph_ll_client.h usr/include/cephfs/libcephfs.h usr/lib/libcephfs.so diff --git a/ceph/debian/radosgw.install b/ceph/debian/radosgw.install index 303791694..1b19d292d 100644 --- a/ceph/debian/radosgw.install +++ b/ceph/debian/radosgw.install @@ -1,7 +1,11 @@ lib/systemd/system/ceph-radosgw* +usr/bin/ceph-diff-sorted usr/bin/radosgw usr/bin/radosgw-es usr/bin/radosgw-object-expirer usr/bin/radosgw-token +usr/bin/rgw-orphan-list usr/lib/libradosgw.so* +usr/share/man/man8/ceph-diff-sorted.8 usr/share/man/man8/radosgw.8 +usr/share/man/man8/rgw-orphan-list.8 diff --git a/ceph/doc/cephadm/adoption.rst b/ceph/doc/cephadm/adoption.rst index f94e46325..9444d5670 100644 --- a/ceph/doc/cephadm/adoption.rst +++ b/ceph/doc/cephadm/adoption.rst @@ -138,5 +138,8 @@ Adoption process # systemctl stop ceph-rgw.target # rm -rf /var/lib/ceph/radosgw/ceph-* + For adopting single-site systems without a realm, see also + :ref:`rgw-multisite-migrate-from-single-site`. + #. Check the ``ceph health detail`` output for cephadm warnings about stray cluster daemons or hosts that are not yet managed. diff --git a/ceph/doc/cephadm/install.rst b/ceph/doc/cephadm/install.rst index ca4ff970d..4459a7092 100644 --- a/ceph/doc/cephadm/install.rst +++ b/ceph/doc/cephadm/install.rst @@ -110,14 +110,18 @@ Enable Ceph CLI =============== Cephadm does not require any Ceph packages to be installed on the -host. However, we recommend enabling easy access to the the ``ceph`` +host. However, we recommend enabling easy access to the ``ceph`` command. There are several ways to do this: * The ``cephadm shell`` command launches a bash shell in a container - with all of the Ceph packages installed. By default, if + with all of the Ceph packages installed. By default, if configuration and keyring files are found in ``/etc/ceph`` on the host, they are passed into the container environment so that the - shell is fully functional:: + shell is fully functional. Note that when executed on a MON host, + ``cephadm shell`` will infer the ``config`` from the MON container + instead of using the default configuration. If ``--mount `` + is given, then the host ```` (file or directory) will appear + under ``/mnt`` inside the container:: # cephadm shell @@ -167,6 +171,8 @@ To add each new host to the cluster, perform two steps: # ceph orch host add host3 +.. _deploy_additional_monitors: + Deploy additional monitors (optional) ===================================== @@ -363,3 +369,6 @@ RADOS pool *nfs-ganesha* and namespace *nfs-ns*,:: See :ref:`orchestrator-cli-placement-spec` for details of the placement specification. +Deploying custom containers +=========================== +It is also possible to choose different containers than the default containers to deploy Ceph. See :ref:`containers` for information about your options in this regard. diff --git a/ceph/doc/cephadm/monitoring.rst b/ceph/doc/cephadm/monitoring.rst index 31c93a3cc..4b6471caf 100644 --- a/ceph/doc/cephadm/monitoring.rst +++ b/ceph/doc/cephadm/monitoring.rst @@ -1,9 +1,9 @@ Monitoring Stack with Cephadm ============================= -The Ceph dashboard makes use of prometheus, grafana, and related tools -to store and visualize detailed metrics on cluster utilization and -performance. Ceph users have three options: +Ceph Dashboard uses `Prometheus `_, `Grafana +`_, and related tools to store and visualize detailed +metrics on cluster utilization and performance. Ceph users have three options: #. Have cephadm deploy and configure these services. This is the default when bootstrapping a new cluster unless the ``--skip-monitoring-stack`` @@ -14,8 +14,27 @@ performance. Ceph users have three options: #. Skip the monitoring stack completely. Some Ceph dashboard graphs will not be available. -Deploying monitoring with cephadm ---------------------------------- +The monitoring stack consists of `Prometheus `_, +Prometheus exporters (:ref:`mgr-prometheus`, `Node exporter +`_), `Prometheus Alert +Manager `_ and `Grafana +`_. + +.. note:: + + Prometheus' security model presumes that untrusted users have access to the + Prometheus HTTP endpoint and logs. Untrusted users have access to all the + (meta)data Prometheus collects that is contained in the database, plus a + variety of operational and debugging information. + + However, Prometheus' HTTP API is limited to read-only operations. + Configurations can *not* be changed using the API and secrets are not + exposed. Moreover, Prometheus has some built-in measures to mitigate the + impact of denial of service attacks. + + Please see `Prometheus' Security model + ` for more detailed + information. By default, bootstrap will deploy a basic monitoring stack. If you did not do this (by passing ``--skip-monitoring-stack``, or if you @@ -57,6 +76,45 @@ completed, you should see something like this from ``ceph orch ls``:: node-exporter 2/2 6s ago docker.io/prom/node-exporter:latest e5a616e4b9cf present prometheus 1/1 6s ago docker.io/prom/prometheus:latest e935122ab143 present +Using custom images +~~~~~~~~~~~~~~~~~~~ + +It is possible to install or upgrade monitoring components based on other +images. To do so, the name of the image to be used needs to be stored in the +configuration first. The following configuration options are available. + +- ``container_image_prometheus`` +- ``container_image_grafana`` +- ``container_image_alertmanager`` +- ``container_image_node_exporter`` + +Custom images can be set with the ``ceph config`` command:: + + ceph config set mgr mgr/cephadm/ + +For example:: + + ceph config set mgr mgr/cephadm/container_image_prometheus prom/prometheus:v1.4.1 + +.. note:: + + By setting a custom image, the default value will be overridden (but not + overwritten). The default value changes when updates become available. + By setting a custom image, you will not be able to update the component + you have set the custom image for automatically. You will need to + manually update the configuration (image name and tag) to be able to + install updates. + + If you choose to go with the recommendations instead, you can reset the + custom image you have set before. After that, the default value will be + used again. Use ``ceph config rm`` to reset the configuration option:: + + ceph config rm mgr mgr/cephadm/ + + For example:: + + ceph config rm mgr mgr/cephadm/container_image_prometheus + Disabling monitoring -------------------- @@ -87,3 +145,10 @@ cluster. * To enable the dashboard's prometheus-based alerting, see :ref:`dashboard-alerting`. * To enable dashboard integration with Grafana, see :ref:`dashboard-grafana`. + +Enabling RBD-Image monitoring +--------------------------------- + +Due to performance reasons, monitoring of RBD images is disabled by default. For more information please see +:ref:`prometheus-rbd-io-statistics`. If disabled, the overview and details dashboards will stay empty in Grafana +and the metrics will not be visible in Prometheus. diff --git a/ceph/doc/cephadm/operations.rst b/ceph/doc/cephadm/operations.rst index 59e2f2965..7768b1ff4 100644 --- a/ceph/doc/cephadm/operations.rst +++ b/ceph/doc/cephadm/operations.rst @@ -254,3 +254,23 @@ You can remove a broken host from management with:: You can disable this health warning with:: ceph config set mgr mgr/cephadm/warn_on_failed_host_check false + +/etc/ceph/ceph.conf +=================== + +Cephadm uses a minimized ``ceph.conf`` that only contains +a minimal set of information to connect to the Ceph cluster. + +To update the configuration settings, use:: + + ceph config set ... + + +To set up an initial configuration before calling +`bootstrap`, create an initial ``ceph.conf`` file. For example:: + + cat < /etc/ceph/ceph.conf + [global] + osd crush chooseleaf type = 0 + EOF + cephadm bootstrap -c /root/ceph.conf ... diff --git a/ceph/doc/cephadm/troubleshooting.rst b/ceph/doc/cephadm/troubleshooting.rst index da4f55317..29e36958e 100644 --- a/ceph/doc/cephadm/troubleshooting.rst +++ b/ceph/doc/cephadm/troubleshooting.rst @@ -148,3 +148,31 @@ To verify that the public key is in the authorized_keys file, run the following [root@mon1 ~]# cephadm shell -- ceph config-key get mgr/cephadm/ssh_identity_pub > key.pub [root@mon1 ~]# grep "`cat key.pub`" /root/.ssh/authorized_keys + +Failed to infer CIDR network error +---------------------------------- + +If you see this error:: + + ERROR: Failed to infer CIDR network for mon ip ***; pass --skip-mon-network to configure it later + +Or this error:: + + Must set public_network config option or specify a CIDR network, ceph addrvec, or plain IP + +This means that you must run a command of this form:: + + ceph config set mon public_network + +For more detail on operations of this kind, see :ref:`deploy_additional_monitors` + +Accessing the admin socket +-------------------------- + +Each Ceph daemon provides an admin socket that bypasses the +MONs (See :ref:`rados-monitoring-using-admin-socket`). + +To access the admin socket, first enter the daemon container on the host:: + + [root@mon1 ~]# cephadm enter --name + [ceph: root@mon1 /]# ceph --admin-daemon /var/run/ceph/ceph-.asok config show diff --git a/ceph/doc/cephfs/administration.rst b/ceph/doc/cephfs/administration.rst index 74e73b698..605864431 100644 --- a/ceph/doc/cephfs/administration.rst +++ b/ceph/doc/cephfs/administration.rst @@ -6,6 +6,9 @@ CephFS Administrative commands File Systems ------------ +.. note:: The names of the file systems, metadata pools, and data pools can + only have characters in the set [a-zA-Z0-9\_-.]. + These commands operate on the CephFS file systems in your Ceph cluster. Note that by default only one file system is permitted: to enable creation of multiple file systems use ``ceph fs flag set enable_multiple true``. diff --git a/ceph/doc/cephfs/createfs.rst b/ceph/doc/cephfs/createfs.rst index 11c54a1cd..241ba3dc9 100644 --- a/ceph/doc/cephfs/createfs.rst +++ b/ceph/doc/cephfs/createfs.rst @@ -35,6 +35,8 @@ Generally, the metadata pool will have at most a few gigabytes of data. For this reason, a smaller PG count is usually recommended. 64 or 128 is commonly used in practice for large clusters. +.. note:: The names of the file systems, metadata pools, and data pools can + only have characters in the set [a-zA-Z0-9\_-.]. Creating a file system ====================== diff --git a/ceph/doc/cephfs/file-layouts.rst b/ceph/doc/cephfs/file-layouts.rst index b779b16d2..6ff834b08 100644 --- a/ceph/doc/cephfs/file-layouts.rst +++ b/ceph/doc/cephfs/file-layouts.rst @@ -20,10 +20,10 @@ Layout fields ------------- pool - String, giving ID or name. Which RADOS pool a file's data objects will be stored in. + String, giving ID or name. String can only have characters in the set [a-zA-Z0-9\_-.]. Which RADOS pool a file's data objects will be stored in. pool_namespace - String. Within the data pool, which RADOS namespace the objects will + String with only characters in the set [a-zA-Z0-9\_-.]. Within the data pool, which RADOS namespace the objects will be written to. Empty by default (i.e. default namespace). stripe_unit diff --git a/ceph/doc/cephfs/fs-volumes.rst b/ceph/doc/cephfs/fs-volumes.rst index f602cc106..f77bd6c88 100644 --- a/ceph/doc/cephfs/fs-volumes.rst +++ b/ceph/doc/cephfs/fs-volumes.rst @@ -115,15 +115,16 @@ FS Subvolumes Create a subvolume using:: - $ ceph fs subvolume create [--size --group_name --pool_layout --uid --gid --mode ] + $ ceph fs subvolume create [--size --group_name --pool_layout --uid --gid --mode --namespace-isolated] The command succeeds even if the subvolume already exists. When creating a subvolume you can specify its subvolume group, data pool layout, uid, gid, file mode in octal numerals, and size in bytes. The size of the subvolume is -specified by setting a quota on it (see :doc:`/cephfs/quota`). By default a -subvolume is created within the default subvolume group, and with an octal file +specified by setting a quota on it (see :doc:`/cephfs/quota`). The subvolume can be +created in a separate RADOS namespace by specifying --namespace-isolated option. By +default a subvolume is created within the default subvolume group, and with an octal file mode '755', uid of its subvolume group, gid of its subvolume group, data pool layout of its parent directory and no size limit. @@ -172,6 +173,7 @@ The output format is json and contains fields as follows. * data_pool: data pool the subvolume belongs to * path: absolute path of a subvolume * type: subvolume type indicating whether it's clone or subvolume +* pool_namespace: RADOS namespace of the subvolume List subvolumes using:: @@ -193,6 +195,18 @@ List snapshots of a subvolume using:: $ ceph fs subvolume snapshot ls [--group_name ] +Fetch the metadata of a snapshot using:: + + $ ceph fs subvolume snapshot info [--group_name ] + +The output format is json and contains fields as follows. + +* created_at: time of creation of snapshot in the format "YYYY-MM-DD HH:MM:SS:ffffff" +* data_pool: data pool the snapshot belongs to +* has_pending_clones: "yes" if snapshot clone is in progress otherwise "no" +* protected: "yes" if snapshot is protected otherwise "no" +* size: snapshot size in bytes + Cloning Snapshots ----------------- diff --git a/ceph/doc/dev/cephadm.rst b/ceph/doc/dev/cephadm.rst index 2fd475f13..1784de422 100644 --- a/ceph/doc/dev/cephadm.rst +++ b/ceph/doc/dev/cephadm.rst @@ -85,3 +85,18 @@ When you're done, you can tear down the cluster with:: sudo ../src/ckill.sh # or, sudo ../src/cephadm/cephadm rm-cluster --force --fsid `cat fsid` + +Note regarding network calls from CLI handlers +============================================== + +Executing any cephadm CLI commands like ``ceph orch ls`` will block +the mon command handler thread within the MGR, thus preventing any +concurrent CLI calls. Note that pressing ``^C`` will not resolve this +situation, as *only* the client will be aborted, but not exceution +itself. This means, cephadm will be completely unresonsive, until the +execution of the CLI handler is fully completed. Note that even +``ceph orch ps`` will not respond, while another handler is executed. + +This means, we should only do very few calls to remote hosts synchronously. +As a guideline, cephadm should do at most ``O(1)`` network calls in CLI handlers. +Everything else should be done asynchronously in other threads, like ``serve()``. diff --git a/ceph/doc/images/dashboard/invalid-credentials.png b/ceph/doc/images/dashboard/invalid-credentials.png new file mode 100644 index 0000000000000000000000000000000000000000..abd78b024407c73ae3bc7ef705658a5fe89bb5b9 GIT binary patch literal 5787 zcmX9?by!s0*S&;@%us@ebW3+hiAYL^Fu)+)(jlD+2n;3Q>kQr9U4wLYgMvf1bi+6P zzCZ4{`<~~VXFuoev-e(WebrD?z{jJ&0{{SDNl{h{05G`GHUt+FJ$7-!{Lrr_&WieO z001KS_h10<6msE|JE$2$ z1p5tcTpYF{%a0!yu|LU8Gwd`=(hkE3g9S!Yyhpy>x+cHc1kVI+DTdJDQoU?=4lWP; zNrMS2IbmZSyjcJEKO(d{NJ3yCpMx8-R}%xt6O38nBzNI zUA$Dknic2_sKH9a!O~i?bVOEKvYOI#AL0%eik=bU>Toa+;g;#j(o56DkrLx#Coo_J z=S2Mf2UGXUpLx~BGM;Hy>Qna?AC!v;4xY?=ADH(b?k@+2UMa^OedXzG?&}`d{Wh9* zaM~M7iJE`YwsQaM{tzjmS8nq=ukNp!L9FxvsOK%B-GNfvKQJSt zYf&UCV#lq0t>GvEwa51rx^sD|+QvHsntZ?NZ^_bsP*(+zSmOZu%QO^-glGP&wW^0w z_*JmE+GTqF1BYR?Z}mVkt~RBOsHe84YXW!D>( zk`!*O?Wh>~wliR!EZwE4w;VZ$@^NKDpJpT!2QfQ`4@C{{@YoOb9 zDd3*^b?T74)i!Fu;}5?>@U0i}H|N_=KTafph?G?Se3f!Bx1Fgmky5C*py>LLZzo2E z*XFsKxq{)`yHLIIe4V0_wkP$02Fpj)JSo@bPEsf04|htRkW&Ij+VR@rn{Pr7@0%LW z&$&J*2eUx?6&O?G6YO*^l!94aMwtBM%8 z2imKmw@Je$u%w#1Ul^i@){k->^gije?7{hQW&hXbd4zJMz(zc*|d+8Ux& zxDy%W;OFN(njqI}#-#7#jMOY{4BX0aFo_$hg!y0KwpUDG-E2`f933tq8Vk(*FipAI zE^Y|2<+ZD76rW2Elb6k(r(Fr=6n9C^^Q&H9Quo=))cRmpP zV8JA-{VWn0jpd{}oZexG$BoN;bHDPGGOe=;(D2wuJP5FzA7=zi_78$(L#`tw{N+G} zsmmpeo^R+I6tMx1Zn@jss(3(pZ09VG)O4|qOp6U5pPB|t3d$TevSbt#fD9e>gGt+| zf?*Es-$Q4Su5vyGXStv{j9+V@O#vSx?PTcUM22^j(K)sxY`6z()TkWs|$tuClj zj1k0vDQ`d*Pset=B5YUlR(9&k=*#zMeS2*lFX^8i!ClhFQ@SdPmQDyM>#6;Lb{R7e zAoEyXKyN(hJ4R6Z2mt`}a(&EQK#0IcseqDJv4u;|Jo}omPWi-xTy$(lfUGXHAGB{*X+<&lsG$dx6!+THC0E?USo7&+uFU+zZpmxV3fJ?+ZDug6wduYLj5 z%&9tPhElAAX)j~Ij8MZxSCFG*8SeuyAWG;LRY4|BNE3n`lyjL#dRJ%Un@#6@#hyQV zWAx)nD2yaP$<1pg?)X?)8yV+*9Q8Z>^#f%6>A}%&LwQ#d^HTAKWvu)BAE38OS z%<%-RQ}XmnSuw%Fao=M-)-7DwK}F5I{arDX(Fv(Tu!MC-1CF^s%hxi--|H-FCYqhI~?@vkDmIy7TZsDa4H{wJRGn%U;sRaxh>m4`G0fGMI5?wH<(uI9T-smp%iu7{bI zLz6#u1IS+03XFEOjBNicujK3Bj`t;;{}LwD23?%}5PQ2i+E7DszdX|WMdn{n4LvW1 z;m@l~Nr1jA^UnIin9(5ugW8dr&mkh6J;^PPw%aTpv&A!6Z1$sk;1VD6=*X3>(Hju( zX9)PZxQAHAW<9b>%iT{W99R*!QvOyWDg<}$Q<}9t{vP;4NuZ);e4-nlv2PXYLvy|L z_ujc!j0_1OIWiLafl^N9ABl=$0oR97#sSImFVl)joM{DOka(^af5`=;4y(Uf%g0uL z_J6MwRHzy*vrY1-r@Q*C?0&ADv%FNCv8iy~*YMw#QA0t{-Ql5e`FRN>;moHcj}AFG zD*zven@y;vYfSeW($GQoVSjIZ|1LOJ1gG$awKaD$jb&{o!kV3xk?j{p?C&%!gwuNj z$w^wQLK`WDJ^IAQ7DT^o{B&+{9%PZvFP2DSb9coIs+zJ0FMzc3w!7$CsCe5|X_k)o zap@Yyjc&O|Mw>Hip10`FKMH3ZO`nD~+v(_uq~SO$Gs<|->-nGY8)dg%cIPxBL@)po zNPn6Ur|RNGL_u4eq4y59k=`fYRReu@PWiZ&7sm$2WI24p&p#KHRJgm;f}Z4hG%bBy zZM39{_#Tw}md#t|vROt~=eY@R&{KgK{YcZvT&q5Q+}grJ`Be7FLO#pVx! zCwu$e0&BHvT)4}#=5CJ)+v2_>-7|M1KHXW5^1M!FFcwW8-yYi}R~m2G=%5j){KBfP z952^P%wX#MjvuJmKZC#Z(m8nWhw4G6>eQD#?{Ea)Z$jo2ch>)mIlpLrlZ(<{c`$iH zo2#K?LDej3d2DReXRov%p-Xth1IkiV*38L;%)M(KGT!1pO>H&|c$2=WR~bAIbtO97 zIl2|9nPR=>qd%Z|L-U6~vvS>*4KOj~nAX$PN_sYPK;z(hK|Dak2TJqTFXAz(06ih` zy)l@3w@65{uwpa4{jJUjYhR{=^JDJ)asV3JbScE7y|Sbvw|M?E41}%bfAh((uBZ^~ z&4t){GAXKLFgO1*^zojMF;V^a!*QxuYDJwDIPp!cEGW>}h;8THJNPU;Tb(U$Exy{MxS7_r_Jen=(3;8vfD=Ye~MX ztknDE9w;E$gr-Ppk4(Dh^B@rhC^WC~?Fdp!Jai0k=#`5HL;I z`FX5e7KjUM+{xQI+x&&YHu5dbBi}^j6XBAmS4S=vOrt{~pw9Peh!-~bUu>ap+Yiln z7w|SwuuX6qsubUT_+kC?OME;&TAp>)HA(7VX^C9Y+DFG(`-YT7F8k{EOuYz5jX#Hh ze6(~;G%3c)d04G_CaXhmi^)6#sXKNSpSrhkJS7GGhvuWB@A}SIrL3rkK7<(eGC*;o7qkQCE-F@89k+|auP=nE|l2CH~hsw3N(bb`4xF?4A zdNswqaC%O`;^khYa6{jT|1Ot*6Z(hewf)+x!EKYQGbA=Bp(9`XM`l6M6NbFOLNPZ31kYw3t+^=o`8E?io~Q32R%YmseFu}tTym=MxS5*<%s`c`=$hyO zf(Fj1dBfO1*o5(|S;F;zWu`eI=9`q$;*Cc;Uw7q{I3j?3p-#^GTxBellJld3f4*9I zqnmP)TU#!bG)SEomyWYkqK6rC5S{ZQFn?rOL!UX+Y%T4zkwObcTtD_X!G1UxUH3C) zyOa{E*k2xXHYNB1E3X0=u1uKQ2YKY6tdR?!=IDz8k}CS<=Xq?o0mUYF*rmVIP5o}Z z@uK^=h@>o=&9s=Cd56K0$M`Xt1Zdt6)NfCRi?2yu70*qT{9Qhu!*Tarj= z;S?z$bz5*gvM)|KTP4VKbjWai>hp)9U=;U)Z`Cg90Dhk&*Q1jr6&#xKO>E>DEpQq>3!oK`3Md2B}tGecGK z1%(4x<@6D#>--g?U5 z4_*S^p@CnFm%T8w4?y}GkCTXr##y%n5Ql>!kVs>@@!}8pcj*Y(Ydkr*9zM$DkptT3 zM?sM#(e@`+E`;ICB_($W8=^gDA75KQynXB}&)MGTGbg?hV>#6|>hr&{ms6f4$bpo! z`E)0m(bDL9T$9=hZtz^kjKB74r?X{r!HtcOgcH{uR&cV6EH!Lwz^WL-A)y!$UA+KW z;A68V$9#QnfiA~rYUY{G3Eyr{4;pPd4!&@>KSN{Su!(QFx=oJ}#)5yxESBbI%{_!x zL`}#CLu?CO-a^R8ae^#g-~e`@Lv%i(Wat6WGg^q_`E;m_euKYK&6Vq~yNMA2E3B{= z_V(q&N@lR0u?I)|#ZwM?(hmsn!3-^oFmLtygrb5mXfq}-(Vn4~6}4s9anX@s(H>%bepA;o)dL^v06~h?|)S+qurY#z~4K zjiOw$a#VT$9`4kIrqBB#zV|TNUNL+YNpH?dC=dX(Jb6Q(Lz4a|{5zqEeo+<-q!D zI#SgdIi`x_Z)!qA0)WWZ``gTnP`wjYP+xyzuAi3+WkJl7*5VNh%Hg{DIGpezS2r)1 z(D5$Cl?PBTGMLLwN~R@VSs&XEp`hBi{~4}l<)`{y7YNcUex|r9Vox$R|3V0kQ^YIc zDDCr+&_y9V=uuR)0)b0JjKhklz&#qKokbp>&N*kQ%s5=xtv3p6#NwAZWeB70Gec9Y z13=)?KE<~o-eGT_GCcvJ$0H=?-ozGZn20(qgkg5_MKcxR$=xBb`zgS{m`UsIpgu41(d^lHGd_7DnB>PE+ThmCP z*`tv-NqL76R8}lxCnS^a38-EOeaJK5`-Jj3AS_px(q3O@AGPvj4uB)w>47;<5Lh4i zReVSM3{fI=r1L)aB8R7m8=A+#)hI0T1fq+Y{UYGGBQnNah^B~j$J|`vJ-gu*{Ac%TV z6e+Z}Q?hO5MBDD{aPP5M9|mdi0D=(3dMKHAhwlKOVca&QCH1e7W22a<>b8l;(UG1p zX%O2*9M@!{5q`2KDC_L{#C&=7ci$#lS+ed<4PsDOZZw?AJ5y(G(w?>6Anm-4C|oa8 zcxb~yw|UxNudA_*!P-ZzXo~%Mj~d+TR06e{vct_ByX(9Ti|xIX?U;znEI_>GG-n+^ zms=Uul4iq#HrK25L$ppdivLdE1I_%}d7Crhf)2kIl|!5Atwl`cI;po-eKEq4Xy%HC zUd`GrsNB#H2brSvi?O6%3TVyEAOS6s9MIl>v(!KM`LgK58l%0&LD10@0~l~a+uWQ~ zz(5QY`@Mr9h%XUd72qYDbM^EmF7upUZ|@2=r^$vjqIoJyxn z;nwQ{?zMsE!X64tPGAnq`^58`@z%G)8CKN#^vHki6g?c zI7o9J(|Y#^yw&`&WZs?jq98a@x`l|2je&?Kj8y-Wl>s}e)>NPDpZ|Z7;^#tLS@fa* z$%$Q9Z2XFmxU%-06`Uiy|Fe|iO zRT~owofLGNH3<2i;Q6VPS#UT`@U8VLQ33J|t?ari7%Mk!ms@r+T~sOa%e}>Mu0hj@lYYQ#&WzULU=L4L6Eq+ln%;P9BBssLDZ@_`Wh XW!QC7gIzybSO%2j)MU$KOhWz-3i>s> literal 0 HcmV?d00001 diff --git a/ceph/doc/install/containers.rst b/ceph/doc/install/containers.rst index c819c646d..2c4efd00e 100644 --- a/ceph/doc/install/containers.rst +++ b/ceph/doc/install/containers.rst @@ -3,6 +3,19 @@ Ceph Container Images ===================== +.. important:: + + Using the ``:latest`` tag is discouraged. If you use the ``:latest`` + tag, there is no guarantee that the same image will be on each of + your hosts. Under these conditions, upgrades might not work + properly. Remember that ``:latest`` is a relative tag, and a moving + target. + + Instead of the ``:latest`` tag, use explicit tags or image IDs. For + example: + + ``podman pull ceph/ceph:v15.2.0`` + Official Releases ----------------- diff --git a/ceph/doc/man/8/CMakeLists.txt b/ceph/doc/man/8/CMakeLists.txt index 02655a8cd..96feaeb6f 100644 --- a/ceph/doc/man/8/CMakeLists.txt +++ b/ceph/doc/man/8/CMakeLists.txt @@ -49,7 +49,9 @@ endif() if(WITH_RADOSGW) list(APPEND man_srcs radosgw.rst - radosgw-admin.rst) + radosgw-admin.rst + rgw-orphan-list.rst + ceph-diff-sorted.rst) endif() if(WITH_RBD) diff --git a/ceph/doc/man/8/ceph-diff-sorted.rst b/ceph/doc/man/8/ceph-diff-sorted.rst new file mode 100644 index 000000000..99e958336 --- /dev/null +++ b/ceph/doc/man/8/ceph-diff-sorted.rst @@ -0,0 +1,71 @@ +:orphan: + +========================================================== + ceph-diff-sorted -- compare two sorted files line by line +========================================================== + +.. program:: ceph-diff-sorted + +Synopsis +======== + +| **ceph-diff-sorted** *file1* *file2* + +Description +=========== + +:program:`ceph-diff-sorted` is a simplifed *diff* utility optimized +for comparing two files with lines that are lexically sorted. + +The output is simplified in comparison to that of the standard `diff` +tool available in POSIX systems. Angle brackets ('<' and '>') are used +to show lines that appear in one file but not the other. The output is +not compatible with the `patch` tool. + +This tool was created in order to perform diffs of large files (e.g., +containing billions of lines) that the standard `diff` tool cannot +handle efficiently. Knowing that the lines are sorted allows this to +be done efficiently with minimal memory overhead. + +The sorting of each file needs to be done lexcially. Most POSIX +systems use the *LANG* environment variable to determine the `sort` +tool's sorting order. To sort lexically we would need something such +as: + + $ LANG=C sort some-file.txt >some-file-sorted.txt + +Examples +======== + +Compare two files:: + + $ ceph-diff-sorted fileA.txt fileB.txt + +Exit Status +=========== + +When complete, the exit status will be set to one of the following: + +0 + files same +1 + files different +2 + usage problem (e.g., wrong number of command-line arguments) +3 + problem opening input file +4 + bad file content (e.g., unsorted order or empty lines) + + +Availability +============ + +:program:`ceph-diff-sorted` is part of Ceph, a massively scalable, +open-source, distributed storage system. Please refer to the Ceph +documentation at http://ceph.com/docs for more information. + +See also +======== + +:doc:`rgw-orphan-list `\(8) diff --git a/ceph/doc/man/8/ceph-osd.rst b/ceph/doc/man/8/ceph-osd.rst index 388e339d9..3b6b2ad2b 100644 --- a/ceph/doc/man/8/ceph-osd.rst +++ b/ceph/doc/man/8/ceph-osd.rst @@ -10,7 +10,7 @@ Synopsis ======== | **ceph-osd** -i *osdnum* [ --osd-data *datapath* ] [ --osd-journal - *journal* ] [ --mkfs ] [ --mkjournal ] [--flush-journal] [--check-allows-journal] [--check-wants-journal] [--check-needs-journal] [ --mkkey ] + *journal* ] [ --mkfs ] [ --mkjournal ] [--flush-journal] [--check-allows-journal] [--check-wants-journal] [--check-needs-journal] [ --mkkey ] [ --osdspec-affinity ] Description @@ -118,6 +118,10 @@ Options Connect to specified monitor (instead of looking through ``ceph.conf``). +.. option:: --osdspec-affinity + + Set an affinity to a certain OSDSpec. + This option can only be used in conjunction with --mkfs. Availability ============ diff --git a/ceph/doc/man/8/ceph.rst b/ceph/doc/man/8/ceph.rst index 4c3407bbd..78aeb5ef2 100644 --- a/ceph/doc/man/8/ceph.rst +++ b/ceph/doc/man/8/ceph.rst @@ -1300,8 +1300,7 @@ Subcommand ``cache-mode`` specifies the caching mode for cache tier . Usage:: - ceph osd tier cache-mode none|writeback|forward|readonly| - readforward|readproxy + ceph osd tier cache-mode writeback|readproxy|readonly|none Subcommand ``remove`` removes the tier (the second one) from base pool (the first one). diff --git a/ceph/doc/man/8/cephadm.rst b/ceph/doc/man/8/cephadm.rst index 9c24d5e82..bd5fa8698 100644 --- a/ceph/doc/man/8/cephadm.rst +++ b/ceph/doc/man/8/cephadm.rst @@ -46,6 +46,8 @@ Synopsis [--config CONFIG] [--keyring KEYRING] command [command ...] +| **cephadm** **unit** [-h] [--fsid FSID] --name NAME command + | **cephadm** **logs** [-h] [--fsid FSID] --name NAME [command [command ...]] | **cephadm** **bootstrap** [-h] [--config CONFIG] [--mon-id MON_ID] @@ -59,12 +61,17 @@ Synopsis | [--initial-dashboard-user INITIAL_DASHBOARD_USER] | [--initial-dashboard-password INITIAL_DASHBOARD_PASSWORD] | [--dashboard-key DASHBOARD_KEY] -| [--dashboard-crt DASHBOARD_CRT] [--skip-mon-network] +| [--dashboard-crt DASHBOARD_CRT] +| [--ssh-config SSH_CONFIG] +| [--ssh-private-key SSH_PRIVATE_KEY] +| [--ssh-public-key SSH_PUBLIC_KEY] [--skip-mon-network] | [--skip-dashboard] [--dashboard-password-noupdate] | [--no-minimize-config] [--skip-ping-check] | [--skip-pull] [--skip-firewalld] [--allow-overwrite] | [--allow-fqdn-hostname] [--skip-prepare-host] | [--orphan-initial-daemons] [--skip-monitoring-stack] +| [--apply-spec APPLY_SPEC] + | **cephadm** **deploy** [-h] --name NAME --fsid FSID [--config CONFIG] @@ -196,6 +203,9 @@ Arguments: * [--initial-dashboard-password INITIAL_DASHBOARD_PASSWORD] Initial password for the initial dashboard user * [--dashboard-key DASHBOARD_KEY] Dashboard key * [--dashboard-crt DASHBOARD_CRT] Dashboard certificate +* [--ssh-config SSH_CONFIG] SSH config +* [--ssh-private-key SSH_PRIVATE_KEY] SSH private key +* [--ssh-public-key SSH_PUBLIC_KEY] SSH public key * [--skip-mon-network] set mon public_network based on bootstrap mon ip * [--skip-dashboard] do not enable the Ceph Dashboard * [--dashboard-password-noupdate] stop forced dashboard password change @@ -208,7 +218,7 @@ Arguments: * [--skip-prepare-host] Do not prepare host * [--orphan-initial-daemons] Do not create initial mon, mgr, and crash service specs * [--skip-monitoring-stack] Do not automatically provision monitoring stack] (prometheus, grafana, alertmanager, node-exporter) - +* [--apply-spec APPLY_SPEC] Apply cluster spec after bootstrap (copy ssh key, add hosts and apply services) ceph-volume ----------- diff --git a/ceph/doc/man/8/radosgw-admin.rst b/ceph/doc/man/8/radosgw-admin.rst index 27569fa53..a80f95d2d 100644 --- a/ceph/doc/man/8/radosgw-admin.rst +++ b/ceph/doc/man/8/radosgw-admin.rst @@ -108,6 +108,11 @@ which are as follows: :command:`bucket rewrite` Rewrite all objects in the specified bucket. +:command:`bucket radoslist` + List the rados objects that contain the data for all objects is + the designated bucket, if --bucket= is specified, or + otherwise all buckets. + :command:`bucket reshard` Reshard a bucket. @@ -399,13 +404,16 @@ which are as follows: Read data log status. :command:`orphans find` - Init and run search for leaked rados objects + Init and run search for leaked rados objects. + DEPRECATED. See the "rgw-orphan-list" tool. :command:`orphans finish` - Clean up search for leaked rados objects + Clean up search for leaked rados objects. + DEPRECATED. See the "rgw-orphan-list" tool. :command:`orphans list-jobs` List the current job-ids for the orphans search. + DEPRECATED. See the "rgw-orphan-list" tool. :command:`role create` create a new AWS role for use with STS. diff --git a/ceph/doc/man/8/rgw-orphan-list.rst b/ceph/doc/man/8/rgw-orphan-list.rst new file mode 100644 index 000000000..408242da2 --- /dev/null +++ b/ceph/doc/man/8/rgw-orphan-list.rst @@ -0,0 +1,69 @@ +:orphan: + +================================================================== + rgw-orphan-list -- list rados objects that are not indexed by rgw +================================================================== + +.. program:: rgw-orphan-list + +Synopsis +======== + +| **rgw-orphan-list** + +Description +=========== + +:program:`rgw-orphan-list` is an *EXPERIMENTAL* RADOS gateway user +administration utility. It produces a listing of rados objects that +are not directly or indirectly referenced through the bucket indexes +on a pool. It places the results and intermediate files on the local +filesystem rather than on the ceph cluster itself, and therefore will +not itself consume additional cluster storage. + +In theory orphans should not exist. However because ceph evolves +rapidly, bugs do crop up, and they may result in orphans that are left +behind. + +In its current form this utility does not take any command-line +arguments or options. It will list the available pools and prompt the +user to enter the pool they would like to list orphans for. + +Behind the scenes it runs `rados ls` and `radosgw-admin bucket +radoslist ...` and produces a list of those entries that appear in the +former but not the latter. Those entries are presumed to be the +orphans. + +Warnings +======== + +This utility is currently considered *EXPERIMENTAL*. + +This utility will produce false orphan entries for unindexed buckets +since such buckets have no bucket indices that can provide the +starting point for tracing. + +Options +======= + +At present there are no options. + +Examples +======== + +Launch the tool:: + + $ rgw-orphan-list + +Availability +============ + +:program:`radosgw-admin` is part of Ceph, a massively scalable, open-source, +distributed storage system. Please refer to the Ceph documentation at +http://ceph.com/docs for more information. + +See also +======== + +:doc:`radosgw-admin `\(8) +:doc:`ceph-diff-sorted `\(8) diff --git a/ceph/doc/man_index.rst b/ceph/doc/man_index.rst index 0c54f32fb..08ef7259a 100644 --- a/ceph/doc/man_index.rst +++ b/ceph/doc/man_index.rst @@ -23,6 +23,7 @@ man/8/ceph-run man/8/ceph-syn man/8/ceph + man/8/cephadm man/8/crushtool man/8/librados-config man/8/monmaptool @@ -40,4 +41,6 @@ man/8/rbd-replay man/8/rbd man/8/rbdmap + man/8/rgw-orphan-list man/8/ceph-immutable-object-cache + man/8/ceph-diff-sorted diff --git a/ceph/doc/mgr/dashboard.rst b/ceph/doc/mgr/dashboard.rst index 5d9cc86f5..be298a569 100644 --- a/ceph/doc/mgr/dashboard.rst +++ b/ceph/doc/mgr/dashboard.rst @@ -65,10 +65,9 @@ aspects of your Ceph cluster: * **Overall cluster health**: Display overall cluster status, performance and capacity metrics. * **Embedded Grafana Dashboards**: Ceph Dashboard is capable of embedding - `Grafana `_ dashboards in many locations, to display - additional information and performance metrics gathered by the - :ref:`mgr-prometheus`. See :ref:`dashboard-grafana` for details on how to - configure this functionality. + `Grafana`_ dashboards in many locations, to display additional information + and performance metrics gathered by the :ref:`mgr-prometheus`. See + :ref:`dashboard-grafana` for details on how to configure this functionality. * **Cluster logs**: Display the latest updates to the cluster's event and audit log files. Log entries can be filtered by priority, date or keyword. * **Hosts**: Display a list of all hosts associated to the cluster, which @@ -378,6 +377,31 @@ The available iSCSI gateways must be defined using the following commands:: Enabling the Embedding of Grafana Dashboards ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +`Grafana`_ requires data from `Prometheus `_. Although +Grafana can use other data sources, the Grafana dashboards we provide contain +queries that are specific to Prometheus. Our Grafana dashboards therefore +require Prometheus as the data source. The Ceph :ref:`mgr-prometheus` also only +exports its data in the Prometheus' common format. The Grafana dashboards rely +on metric names from the Prometheus module and `Node exporter +`_. The Node exporter is a +separate application that provides machine metrics. + +.. note:: + + Prometheus' security model presumes that untrusted users have access to the + Prometheus HTTP endpoint and logs. Untrusted users have access to all the + (meta)data Prometheus collects that is contained in the database, plus a + variety of operational and debugging information. + + However, Prometheus' HTTP API is limited to read-only operations. + Configurations can *not* be changed using the API and secrets are not + exposed. Moreover, Prometheus has some built-in measures to mitigate the + impact of denial of service attacks. + + Please see `Prometheus' Security model + ` for more detailed + information. + Grafana and Prometheus are likely going to be bundled and installed by some orchestration tools along Ceph in the near future, but currently, you will have to install and configure both manually. After you have installed Prometheus and @@ -1068,5 +1092,172 @@ Plug-ins Dashboard Plug-ins extend the functionality of the dashboard in a modular and loosely coupled fashion. +.. _Grafana: https://grafana.com/ + .. include:: dashboard_plugins/feature_toggles.inc.rst .. include:: dashboard_plugins/debug.inc.rst + + +Troubleshooting the Dashboard +----------------------------- + +Locating the Dashboard +^^^^^^^^^^^^^^^^^^^^^^ + +If you are unsure of the location of the Ceph Dashboard, run the following command:: + + $ ceph mgr services | jq .dashboard + "https://host:port" + +The command returns the URL where the Ceph Dashboard is located: ``https://:/`` + +.. note:: + + Many Ceph command line tools return results in JSON format. You may have to install + the `jq `_ command-line JSON processor utility on + your operating system beforehand. + + +Accessing the Dashboard +^^^^^^^^^^^^^^^^^^^^^^^ + +If you are unable to access the Ceph Dashboard, run through the following +commands: + +#. Verify the Ceph Dashboard module is enabled:: + + $ ceph mgr module ls | jq .enabled_modules + + Ensure the Ceph Dashboard module is listed in the return value of the + command. Example snipped output from the command above:: + + [ + "dashboard", + "iostat", + "restful" + ] + +#. If it is not listed, activate the module with the following command:: + + $ ceph mgr module enable dashboard + +#. Check the Ceph Dashboard and/or mgr log file for any errors. The exact + location of the log files depends on the Ceph configuration. + + * Check if mgr log messages are written to a file by:: + + $ ceph config get mgr log_to_file + true + + * Get the location of the log file (it's ``/var/log/ceph/-.log`` + by default):: + + $ ceph config get mgr log_file + /var/log/ceph/$cluster-$name.log + +#. Ensure the SSL/TSL support is configured properly: + + * Check if the SSL/TSL support is enabled:: + + $ ceph config get mgr mgr/dashboard/ssl + + * If the command returns ``true``, verify a certificate exists by:: + + $ ceph config-key get mgr/dashboard/crt + + and:: + + $ ceph config-key get mgr/dashboard/key + + * If it doesn't, run the following command to generate a self-signed + certificate or follow the instructions outlined in + :ref:`dashboard-ssl-tls-support`:: + + $ ceph dashboard create-self-signed-cert + + +Trouble Logging into the Dashboard +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you are unable to log into the Ceph Dashboard and you receive the following +error, run through the procedural checks below: + +.. image:: ../images/dashboard/invalid-credentials.png + :align: center + +#. Check that your user credentials are correct. If you are seeing the + notification message above when trying to log into the Ceph Dashboard, it + is likely you are using the wrong credentials. Double check your username + and password, and ensure the caps lock key is not enabled by accident. + +#. If your user credentials are correct, but you are experiencing the same + error, check that the user account exists:: + + $ ceph dashboard ac-user-show + + This command returns your user data. If the user does not exist, it will + print:: + + $ Error ENOENT: User does not exist + +#. Check if the user is enabled:: + + $ ceph dashboard ac-user-show | jq .enabled + true + + Check if ``enabled`` is set to ``true`` for your user. If not the user is + not enabled, run:: + + $ ceph dashboard ac-user-enable + +Please see :ref:`dashboard-user-role-management` for more information. + + +A Dashboard Feature is Not Working +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When an error occurs on the backend, you will usually receive an error +notification on the frontend. Run through the following scenarios to debug. + +#. Check the Ceph Dashboard/mgr logfile(s) for any errors. These can be + identified by searching for keywords, such as *500 Internal Server Error*, + followed by ``traceback``. The end of a traceback contains more details about + what exact error occurred. +#. Check your web browser's Javascript Console for any errors. + + +Ceph Dashboard Logs +^^^^^^^^^^^^^^^^^^^ + +Dashboard Debug Flag +'''''''''''''''''''' + +With this flag enabled, traceback of errors are included in backend responses. + +To enable this flag via the Ceph Dashboard, navigate from *Cluster* to *Manager +modules*. Select *Dashboard module* and click the edit button. Click the +*debug* checkbox and update. + +To enable it via the CLI, run the following command:: + + $ ceph dashboard debug enable + + +Setting Logging Level of Dashboard Module +''''''''''''''''''''''''''''''''''''''''' + +Setting the logging level to debug makes the log more verbose and helpful for +debugging. + +#. Increase the logging level of manager daemons:: + + $ ceph tell mgr config set debug_mgr 20 + +#. Adjust the logging level of the Ceph Dashboard module via the Dashboard or + CLI: + + * Navigate from *Cluster* to *Manager modules*. Select *Dashboard module* + and click the edit button. Modify the ``log_level`` configuration. + * To adjust it via the CLI, run the following command:: + + $ bin/ceph config set mgr mgr/dashboard/log_level debug diff --git a/ceph/doc/mgr/orchestrator.rst b/ceph/doc/mgr/orchestrator.rst index 440961f1b..6d3fc235f 100644 --- a/ceph/doc/mgr/orchestrator.rst +++ b/ceph/doc/mgr/orchestrator.rst @@ -70,8 +70,34 @@ List hosts associated with the cluster:: Add and remove hosts:: - ceph orch host add - ceph orch host rm + ceph orch host add [] [...] + ceph orch host rm + +Host Specification +------------------ + +Many hosts can be added at once using +``ceph orch apply -i`` by submitting a multi-document YAML file:: + + --- + service_type: host + addr: node-00 + hostname: node-00 + labels: + - example1 + - example2 + --- + service_type: host + addr: node-01 + hostname: node-01 + labels: + - grafana + --- + service_type: host + addr: node-02 + hostname: node-02 + +This can be combined with service specifications (below) to create a cluster spec file to deploy a whole cluster in one command. see ``cephadm bootstrap --apply-spec`` also to do this during bootstrap. Cluster SSH Keys must be copied to hosts prior. OSD Management ============== @@ -88,58 +114,144 @@ filtered to a particular host: Example:: - # ceph orch device ls - Host 192.168.121.206: - Device Path Type Size Rotates Available Model - /dev/sdb hdd 50.0G True True ATA/QEMU HARDDISK - /dev/sda hdd 50.0G True False ATA/QEMU HARDDISK + HOST PATH TYPE SIZE DEVICE AVAIL REJECT REASONS + master /dev/vda hdd 42.0G False locked + node1 /dev/vda hdd 42.0G False locked + node1 /dev/vdb hdd 8192M 387836 False locked, LVM detected, Insufficient space (<5GB) on vgs + node1 /dev/vdc hdd 8192M 450575 False locked, LVM detected, Insufficient space (<5GB) on vgs + node3 /dev/vda hdd 42.0G False locked + node3 /dev/vdb hdd 8192M 395145 False LVM detected, locked, Insufficient space (<5GB) on vgs + node3 /dev/vdc hdd 8192M 165562 False LVM detected, locked, Insufficient space (<5GB) on vgs + node2 /dev/vda hdd 42.0G False locked + node2 /dev/vdb hdd 8192M 672147 False LVM detected, Insufficient space (<5GB) on vgs, locked + node2 /dev/vdc hdd 8192M 228094 False LVM detected, Insufficient space (<5GB) on vgs, locked - Host 192.168.121.181: - Device Path Type Size Rotates Available Model - /dev/sdb hdd 50.0G True True ATA/QEMU HARDDISK - /dev/sda hdd 50.0G True False ATA/QEMU HARDDISK -.. note:: - Output form Ansible orchestrator + + +Erase Devices (Zap Devices) +--------------------------- + +Erase (zap) a device so that it can be resued. ``zap`` calls ``ceph-volume zap`` on the remote host. + +:: + + orch device zap + +Example command:: + + ceph orch device zap my_hostname /dev/sdx + Create OSDs ----------- Create OSDs on a group of devices on a single host:: - ceph orch osd create : - ceph orch osd create -i + ceph orch daemon add osd :device1,device2 + +or:: + ceph orch apply osd -i -The output of ``osd create`` is not specified and may vary between orchestrator backends. -Where ``drive.group.json`` is a JSON file containing the fields defined in -:class:`ceph.deployment_utils.drive_group.DriveGroupSpec` +or:: + + ceph orch apply osd --all-available-devices + + +For a more in-depth guide to DriveGroups please refer to :ref:`drivegroups` Example:: - # ceph orch osd create 192.168.121.206:/dev/sdc - {"status": "OK", "msg": "", "data": {"event": "playbook_on_stats", "uuid": "7082f3ba-f5b7-4b7c-9477-e74ca918afcb", "stdout": "\r\nPLAY RECAP *********************************************************************\r\n192.168.121.206 : ok=96 changed=3 unreachable=0 failed=0 \r\n", "counter": 932, "pid": 10294, "created": "2019-05-28T22:22:58.527821", "end_line": 1170, "runner_ident": "083cad3c-8197-11e9-b07a-2016b900e38f", "start_line": 1166, "event_data": {"ignored": 0, "skipped": {"192.168.121.206": 186}, "ok": {"192.168.121.206": 96}, "artifact_data": {}, "rescued": 0, "changed": {"192.168.121.206": 3}, "pid": 10294, "dark": {}, "playbook_uuid": "409364a6-9d49-4e44-8b7b-c28e5b3adf89", "playbook": "add-osd.yml", "failures": {}, "processed": {"192.168.121.206": 1}}, "parent_uuid": "409364a6-9d49-4e44-8b7b-c28e5b3adf89"}} + # ceph orch daemon add osd node1:/dev/vdd + Created osd(s) 6 on host 'node1' + + +If the 'apply' method is used. You will be presented with a preview of what will happen. + +Example:: + + # ceph orch apply osd --all-available-devices + NAME HOST DATA DB WAL + all-available-devices node1 /dev/vdb - - + all-available-devices node2 /dev/vdc - - + all-available-devices node3 /dev/vdd - - + .. note:: - Output form Ansible orchestrator + Output form Cephadm orchestrator -Decommission an OSD +Remove an OSD ------------------- :: - ceph orch osd rm  [osd-id...] + ceph orch osd rm ... [--replace] [--force] -Removes one or more OSDs from the cluster and the host, if the OSDs are marked as -``destroyed``. +Removes one or more OSDs from the cluster. Example:: # ceph orch osd rm 4 - {"status": "OK", "msg": "", "data": {"event": "playbook_on_stats", "uuid": "1a16e631-906d-48e0-9e24-fa7eb593cc0a", "stdout": "\r\nPLAY RECAP *********************************************************************\r\n192.168.121.158 : ok=2 changed=0 unreachable=0 failed=0 \r\n192.168.121.181 : ok=2 changed=0 unreachable=0 failed=0 \r\n192.168.121.206 : ok=2 changed=0 unreachable=0 failed=0 \r\nlocalhost : ok=31 changed=8 unreachable=0 failed=0 \r\n", "counter": 240, "pid": 10948, "created": "2019-05-28T22:26:09.264012", "end_line": 308, "runner_ident": "8c093db0-8197-11e9-b07a-2016b900e38f", "start_line": 301, "event_data": {"ignored": 0, "skipped": {"localhost": 37}, "ok": {"192.168.121.181": 2, "192.168.121.158": 2, "192.168.121.206": 2, "localhost": 31}, "artifact_data": {}, "rescued": 0, "changed": {"localhost": 8}, "pid": 10948, "dark": {}, "playbook_uuid": "a12ec40e-bce9-4bc9-b09e-2d8f76a5be02", "playbook": "shrink-osd.yml", "failures": {}, "processed": {"192.168.121.181": 1, "192.168.121.158": 1, "192.168.121.206": 1, "localhost": 1}}, "parent_uuid": "a12ec40e-bce9-4bc9-b09e-2d8f76a5be02"}} + Scheduled OSD(s) for removal + + +OSDs that are not safe-to-destroy will be rejected. + +You can query the state of the operation with:: + + # ceph orch osd rm status + NAME HOST PGS STARTED_AT + osd.7 node1 55 2020-04-22 19:28:38.785761 + osd.5 node3 3 2020-04-22 19:28:34.201685 + osd.3 node2 0 2020-04-22 19:28:34.201695 + + +When no PGs are left on the osd, it will be decommissioned and removed from the cluster. + + +Replace an OSD +------------------- +:: + + orch osd rm ... --replace [--force] + +Example:: + + # ceph orch osd rm 4 --replace + Scheduled OSD(s) for replacement + + +This follows the same procedure as the "Remove OSD" part with the exception that the OSD is not permanently removed +from the crush hierarchy, but is assigned a 'destroyed' flag. + +**Preserving the OSD ID** + +The previously set the 'destroyed' flag is used to determined osd ids that will be reused in the next osd deployment. + +If you use OSDSpecs for osd deployment, your newly added disks will be assigned with the osd ids of their replaced +counterpart, granted the new disk still match the OSDSpecs. + +For assistance in this process you can use the 'preview' feature: + +Example:: + + + ceph orch apply osd --service-name --preview + NAME HOST DATA DB WAL + node1 /dev/vdb - - + +Tip: The name of your OSDSpec can be retrieved from **ceph orch ls** + +Alternatively, you can use your OSDSpec file:: + + ceph orch apply osd -i --preview + NAME HOST DATA DB WAL + node1 /dev/vdb - - + + +If this matches your anticipated behavior, just omit the --preview flag to execute the deployment. -.. note:: - Output form Ansible orchestrator .. Blink Device Lights @@ -294,6 +406,7 @@ to specify the deployment of services. For example: - host2 - host3 spec: ... + unmanaged: false Where the properties of a service specification are the following: @@ -305,6 +418,10 @@ Where the properties of a service specification are the following: * ``service_id`` is the name of the service. Omit the service time * ``placement`` is a :ref:`orchestrator-cli-placement-spec` * ``spec``: additional specifications for a specific service. +* ``unmanaged``: If set to ``true``, the orchestrator will not deploy nor + remove any daemon associated with this service. Placement and all other + properties will be ignored. This is useful, if this service should not + be managed temporarily. Each service type can have different requirements for the spec. @@ -462,8 +579,7 @@ Or with hosts: hosts: - host1 - host2 - - host3 - + - host3 Configuring the Orchestrator CLI ================================ @@ -498,7 +614,7 @@ This is an overview of the current implementation status of the orchestrators. =================================== ====== ========= Command Rook Cephadm =================================== ====== ========= - apply iscsi ⚪ ⚪ + apply iscsi ⚪ ✔ apply mds ✔ ✔ apply mgr ⚪ ✔ apply mon ✔ ✔ @@ -513,10 +629,9 @@ This is an overview of the current implementation status of the orchestrators. daemon {stop,start,...} ⚪ ✔ device {ident,fault}-(on,off} ⚪ ✔ device ls ✔ ✔ - iscsi add ⚪ ⚪ + iscsi add ⚪ ✔ mds add ✔ ✔ nfs add ✔ ✔ - ps ⚪ ✔ rbd-mirror add ⚪ ✔ rgw add ✔ ✔ ps ✔ ✔ diff --git a/ceph/doc/mgr/prometheus.rst b/ceph/doc/mgr/prometheus.rst index e01196bb0..8dbc44f52 100644 --- a/ceph/doc/mgr/prometheus.rst +++ b/ceph/doc/mgr/prometheus.rst @@ -31,6 +31,8 @@ configurable with ``ceph config-key set``, with keys ``mgr/prometheus/server_addr`` and ``mgr/prometheus/server_port``. This port is registered with Prometheus's `registry `_. +.. _prometheus-rbd-io-statistics: + RBD IO statistics ----------------- @@ -41,6 +43,10 @@ configuration parameter. The parameter is a comma or space separated list of ``pool[/namespace]`` entries. If the namespace is not specified the statistics are collected for all namespaces in the pool. +Example to activate the RBD-enabled pools ``pool1``, ``pool2`` and ``poolN``:: + + ceph config set mgr mgr/prometheus/rbd_stats_pools "pool1,pool2,poolN" + The module makes the list of all available images scanning the specified pools and namespaces and refreshes it periodically. The period is configurable via the ``mgr/prometheus/rbd_stats_pools_refresh_interval`` @@ -48,6 +54,10 @@ parameter (in sec) and is 300 sec (5 minutes) by default. The module will force refresh earlier if it detects statistics from a previously unknown RBD image. +Example to turn up the sync interval to 10 minutes:: + + ceph config set mgr mgr/prometheus/rbd_stats_pools_refresh_interval 600 + Statistic names and labels ========================== diff --git a/ceph/doc/mgr/telemetry.rst b/ceph/doc/mgr/telemetry.rst index 3e9292be5..6eaaa5c44 100644 --- a/ceph/doc/mgr/telemetry.rst +++ b/ceph/doc/mgr/telemetry.rst @@ -7,10 +7,15 @@ The telemetry module sends anonymous data about the cluster back to the Ceph developers to help understand how Ceph is used and what problems users may be experiencing. +This data is visualized on `public dashboards `_ +that allow the community to quickly see summary statistics on how many clusters +are reporting, their total capacity and OSD count, and version distribution +trends. + Channels -------- -The telemetry report is broken down into several "channels," each with +The telemetry report is broken down into several "channels", each with a different type of information. Assuming telemetry has been enabled, individual channels can be turned on and off. (If telemetry is off, the per-channel setting has no effect.) @@ -18,7 +23,7 @@ the per-channel setting has no effect.) * **basic** (default: on): Basic information about the cluster - capacity of the cluster - - number of monitors, managers, OSDs, MDSs, radosgws, or other daemons + - number of monitors, managers, OSDs, MDSs, object gateways, or other daemons - software version currently being used - number and types of RADOS pools and CephFS file systems - names of configuration options that have been changed from their @@ -50,7 +55,7 @@ deployed, the version of Ceph, the distribution of the hosts and other parameters which help the project to gain a better understanding of the way Ceph is used. -Data is sent over HTTPS to *telemetry.ceph.com*. +Data is sent secured to *https://telemetry.ceph.com*. Sample report ------------- diff --git a/ceph/doc/rados/operations/devices.rst b/ceph/doc/rados/operations/devices.rst index 8e120446a..98691a043 100644 --- a/ceph/doc/rados/operations/devices.rst +++ b/ceph/doc/rados/operations/devices.rst @@ -25,6 +25,27 @@ location and how it is being consumed with:: ceph device info +Identifying physical devices +---------------------------- + +You can blink the drive LEDs on hardware enclosures to make the replacement of +failed disks easy and less error-prone. Use the following command:: + + device light on|off [ident|fault] [--force] + +The ```` parameter is the device identification. You can obtain this +information using the following command:: + + ceph device ls + +The ``[ident|fault]`` parameter is used to set the kind of light to blink. +By default, the `identification` light is used. + +.. note:: + This command needs the Cephadm or the Rook `orchestrator `_ module enabled. + The orchestrator module enabled is shown by executing the following command:: + + ceph orch status Enabling monitoring ------------------- diff --git a/ceph/doc/rados/operations/monitoring.rst b/ceph/doc/rados/operations/monitoring.rst index e3a3e50b4..ce2040a83 100644 --- a/ceph/doc/rados/operations/monitoring.rst +++ b/ceph/doc/rados/operations/monitoring.rst @@ -499,6 +499,7 @@ For a detailed discussion, refer to `Monitoring OSDs and Placement Groups`_. .. _Monitoring OSDs and Placement Groups: ../monitoring-osd-pg +.. _rados-monitoring-using-admin-socket: Using the Admin Socket ====================== diff --git a/ceph/doc/radosgw/config-ref.rst b/ceph/doc/radosgw/config-ref.rst index b16e8dbc5..c9785c6b7 100644 --- a/ceph/doc/radosgw/config-ref.rst +++ b/ceph/doc/radosgw/config-ref.rst @@ -447,6 +447,15 @@ configuration parameters. :Type: Integer :Default: ``3600`` + +``rgw gc max concurrent io`` + +:Description: The maximum number of concurrent IO operations that the RGW garbage + collection thread will use when purging old data. +:Type: Integer +:Default: ``10`` + + Multisite Settings ================== diff --git a/ceph/doc/radosgw/index.rst b/ceph/doc/radosgw/index.rst index 957c3641d..819986926 100644 --- a/ceph/doc/radosgw/index.rst +++ b/ceph/doc/radosgw/index.rst @@ -69,6 +69,7 @@ you may write data with one API and retrieve it with the other. STS Lite Keycloak Role + Orphan List and Associated Tooliing troubleshooting Manpage radosgw <../../man/8/radosgw> Manpage radosgw-admin <../../man/8/radosgw-admin> diff --git a/ceph/doc/radosgw/multisite.rst b/ceph/doc/radosgw/multisite.rst index b5e1b4cd9..d39632341 100644 --- a/ceph/doc/radosgw/multisite.rst +++ b/ceph/doc/radosgw/multisite.rst @@ -641,6 +641,8 @@ If the former master zone recovers, revert the operation. # systemctl restart ceph-radosgw@rgw.`hostname -s` +.. _rgw-multisite-migrate-from-single-site: + Migrating a Single Site System to Multi-Site ============================================ diff --git a/ceph/doc/radosgw/notifications.rst b/ceph/doc/radosgw/notifications.rst index fa608a409..3c0945f5c 100644 --- a/ceph/doc/radosgw/notifications.rst +++ b/ceph/doc/radosgw/notifications.rst @@ -67,7 +67,7 @@ To update a topic, use the same command used for topic creation, with the topic &Name= &push-endpoint= [&Attributes.entry.1.key=amqp-exchange&Attributes.entry.1.value=] - [&Attributes.entry.2.key=amqp-ack-level&Attributes.entry.2.value=none|broker] + [&Attributes.entry.2.key=amqp-ack-level&Attributes.entry.2.value=none|broker|routable] [&Attributes.entry.3.key=verify-ssl&Attributes.entry.3.value=true|false] [&Attributes.entry.4.key=kafka-ack-level&Attributes.entry.4.value=none|broker] [&Attributes.entry.5.key=use-ssl&Attributes.entry.5.value=true|false] @@ -93,10 +93,11 @@ Request parameters: - port defaults to: 5672 - vhost defaults to: "/" - amqp-exchange: the exchanges must exist and be able to route messages based on topics (mandatory parameter for AMQP0.9.1) - - amqp-ack-level: no end2end acking is required, as messages may persist in the broker before delivered into their final destination. Two ack methods exist: + - amqp-ack-level: no end2end acking is required, as messages may persist in the broker before delivered into their final destination. Three ack methods exist: - "none": message is considered "delivered" if sent to broker - "broker": message is considered "delivered" if acked by broker (default) + - "routable": message is considered "delivered" if broker can route to a consumer - Kafka endpoint diff --git a/ceph/doc/radosgw/orphans.rst b/ceph/doc/radosgw/orphans.rst new file mode 100644 index 000000000..9a77d60de --- /dev/null +++ b/ceph/doc/radosgw/orphans.rst @@ -0,0 +1,115 @@ +================================== +Orphan List and Associated Tooling +================================== + +.. version added:: Luminous + +.. contents:: + +Orphans are RADOS objects that are left behind after their associated +RGW objects are removed. Normally these RADOS objects are removed +automatically, either immediately or through a process known as +"garbage collection". Over the history of RGW, however, there may have +been bugs that prevented these RADOS objects from being deleted, and +these RADOS objects may be consuming space on the Ceph cluster without +being of any use. From the perspective of RGW, we call such RADOS +objects "orphans". + +Orphans Find -- DEPRECATED +-------------------------- + +The `radosgw-admin` tool has/had three subcommands to help manage +orphans, however these subcommands are (or will soon be) +deprecated. These subcommands are: + +:: + # radosgw-admin orphans find ... + # radosgw-admin orphans finish ... + # radosgw-admin orphans list-jobs ... + +There are two key problems with these subcommands, however. First, +these subcommands have not been actively maintained and therefore have +not tracked RGW as it has evolved in terms of features and updates. As +a result the confidence that these subcommands can accurately identify +true orphans is presently low. + +Second, these subcommands store intermediate results on the cluster +itself. This can be problematic when cluster administrators are +confronting insufficient storage space and want to remove orphans as a +means of addressing the issue. The intermediate results could strain +the existing cluster storage capacity even further. + +For these reasons "orphans find" has been deprecated. + +Orphan List +----------- + +Because "orphans find" has been deprecated, RGW now includes an +additional tool -- 'rgw-orphan-list'. When run it will list the +available pools and prompt the user to enter the name of the data +pool. At that point the tool will, perhaps after an extended period of +time, produce a local file containing the RADOS objects from the +designated pool that appear to be orphans. The administrator is free +to examine this file and the decide on a course of action, perhaps +removing those RADOS objects from the designated pool. + +All intermediate results are stored on the local file system rather +than the Ceph cluster. So running the 'rgw-orphan-list' tool should +have no appreciable impact on the amount of cluster storage consumed. + +WARNING: Experimental Status +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The 'rgw-orphan-list' tool is new and therefore currently considered +experimental. The list of orphans produced should be "sanity checked" +before being used for a large delete operation. + +WARNING: Specifying a Data Pool +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If a pool other than an RGW data pool is specified, the results of the +tool will be erroneous. All RADOS objects found on such a pool will +falsely be designated as orphans. + +WARNING: Unindexed Buckets +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +RGW allows for unindexed buckets, that is buckets that do not maintain +an index of their contents. This is not a typical configuration, but +it is supported. Because the 'rgw-orphan-list' tool uses the bucket +indices to determine what RADOS objects should exist, objects in the +unindexed buckets will falsely be listed as orphans. + + +RADOS List +---------- + +One of the sub-steps in computing a list of orphans is to map each RGW +object into its corresponding set of RADOS objects. This is done using +a subcommand of 'radosgw-admin'. + +:: + # radosgw-admin bucket radoslist [--bucket={bucket-name}] + +The subcommand will produce a list of RADOS objects that support all +of the RGW objects. If a bucket is specified then the subcommand will +only produce a list of RADOS objects that correspond back the RGW +objects in the specified bucket. + +Note: Shared Bucket Markers +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Some administrators will be aware of the coding schemes used to name +the RADOS objects that correspond to RGW objects, which include a +"marker" unique to a given bucket. + +RADOS objects that correspond with the contents of one RGW bucket, +however, may contain a marker that specifies a different bucket. This +behavior is a consequence of the "shallow copy" optimization used by +RGW. When larger objects are copied from bucket to bucket, only the +"head" objects are actually copied, and the tail objects are +shared. Those shared objects will contain the marker of the original +bucket. + +.. _Data Layout in RADOS : ../layout +.. _Pool Placement and Storage Classes : ../placement diff --git a/ceph/doc/radosgw/pubsub-module.rst b/ceph/doc/radosgw/pubsub-module.rst index 61cd4def2..fd3b9f021 100644 --- a/ceph/doc/radosgw/pubsub-module.rst +++ b/ceph/doc/radosgw/pubsub-module.rst @@ -150,7 +150,7 @@ To update a topic, use the same command used for topic creation, with the topic :: - PUT /topics/[?OpaqueData=][&push-endpoint=[&amqp-exchange=][&amqp-ack-level=none|broker][&verify-ssl=true|false][&kafka-ack-level=none|broker][&use-ssl=true|false][&ca-location=]] + PUT /topics/[?OpaqueData=][&push-endpoint=[&amqp-exchange=][&amqp-ack-level=none|broker|routable][&verify-ssl=true|false][&kafka-ack-level=none|broker][&use-ssl=true|false][&ca-location=]] Request parameters: @@ -173,10 +173,11 @@ The endpoint URI may include parameters depending with the type of endpoint: - port defaults to: 5672 - vhost defaults to: "/" - amqp-exchange: the exchanges must exist and be able to route messages based on topics (mandatory parameter for AMQP0.9.1) - - amqp-ack-level: no end2end acking is required, as messages may persist in the broker before delivered into their final destination. Two ack methods exist: + - amqp-ack-level: no end2end acking is required, as messages may persist in the broker before delivered into their final destination. Three ack methods exist: - "none": message is considered "delivered" if sent to broker - "broker": message is considered "delivered" if acked by broker (default) + - "routable": message is considered "delivered" if broker can route to a consumer - Kafka endpoint @@ -348,7 +349,7 @@ Creates a new subscription. :: - PUT /subscriptions/?topic=[?push-endpoint=[&amqp-exchange=][&amqp-ack-level=none|broker][&verify-ssl=true|false][&kafka-ack-level=none|broker][&ca-location=]] + PUT /subscriptions/?topic=[?push-endpoint=[&amqp-exchange=][&amqp-ack-level=none|broker|routable][&verify-ssl=true|false][&kafka-ack-level=none|broker][&ca-location=]] Request parameters: @@ -370,10 +371,11 @@ The endpoint URI may include parameters depending with the type of endpoint: - port defaults to: 5672 - vhost defaults to: "/" - amqp-exchange: the exchanges must exist and be able to route messages based on topics (mandatory parameter for AMQP0.9.1) - - amqp-ack-level: no end2end acking is required, as messages may persist in the broker before delivered into their final destination. Two ack methods exist: + - amqp-ack-level: no end2end acking is required, as messages may persist in the broker before delivered into their final destination. Three ack methods exist: - "none": message is considered "delivered" if sent to broker - "broker": message is considered "delivered" if acked by broker (default) + - "routable": message is considered "delivered" if broker can route to a consumer - Kafka endpoint diff --git a/ceph/fusetrace/fusetrace_ll.cc b/ceph/fusetrace/fusetrace_ll.cc index b411a83fe..f10c29fd2 100644 --- a/ceph/fusetrace/fusetrace_ll.cc +++ b/ceph/fusetrace/fusetrace_ll.cc @@ -11,8 +11,6 @@ gcc -Wall `pkg-config fuse --cflags --libs` -lulockmgr fusexmp_fh.c -o fusexmp_fh */ -#define FUSE_USE_VERSION 30 - #ifdef HAVE_CONFIG_H #include #endif diff --git a/ceph/monitoring/grafana/dashboards/radosgw-sync-overview.json b/ceph/monitoring/grafana/dashboards/radosgw-sync-overview.json new file mode 100644 index 000000000..e9136d78e --- /dev/null +++ b/ceph/monitoring/grafana/dashboards/radosgw-sync-overview.json @@ -0,0 +1,440 @@ +{ + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1534386107523, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (source_zone) (rate(ceph_data_sync_from_zone_fetch_bytes_sum[30s]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{source_zone}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Replication (throughput) from Source Zone", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "unit": "bytes", + "format": "Bps", + "decimals": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 7, + "w": 7.4, + "x": 8.3, + "y": 0 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (source_zone) (rate(ceph_data_sync_from_zone_fetch_bytes_count[30s]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{source_zone}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Replication (objects) from Source Zone", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "decimals": null, + "label": "Objects/s", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (source_zone) (rate(ceph_data_sync_from_zone_poll_latency_sum[30s]) * 1000)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{source_zone}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Polling Request Latency from Source Zone", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "unit": "s", + "format": "ms", + "decimals": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (source_zone) (rate(ceph_data_sync_from_zone_fetch_errors[30s]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{source_zone}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Unsuccessful Object Replications from Source Zone", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "decimals": null, + "label": "Count/s", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "refresh": "15s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "overview" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 2, + "includeAll": true, + "label": null, + "multi": false, + "name": "rgw_servers", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "tags": [], + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "15s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "RGW Sync Overview", + "uid": "rgw-sync-overview", + "version": 2 +} diff --git a/ceph/monitoring/grafana/dashboards/rbd-details.json b/ceph/monitoring/grafana/dashboards/rbd-details.json new file mode 100644 index 000000000..68fffad1e --- /dev/null +++ b/ceph/monitoring/grafana/dashboards/rbd-details.json @@ -0,0 +1,409 @@ +{ + "__inputs": [], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.3.3" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Detailed Performance of RBD Images (IOPS/Throughput/Latency)", + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1584428820779, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$Datasource", + "fill": 1, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(ceph_rbd_write_ops{pool=\"$Pool\", image=\"$Image\"}[30s])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "irate(ceph_rbd_read_ops{pool=\"$Pool\", image=\"$Image\"}[30s])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "IOPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "iops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": true, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$Datasource", + "fill": 1, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(ceph_rbd_write_bytes{pool=\"$Pool\", image=\"$Image\"}[30s])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "irate(ceph_rbd_read_bytes{pool=\"$Pool\", image=\"$Image\"}[30s])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Throughput", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": true, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$Datasource", + "fill": 1, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(ceph_rbd_write_latency_sum{pool=\"$Pool\", image=\"$Image\"}[30s]) / irate(ceph_rbd_write_latency_count{pool=\"$Pool\", image=\"$Image\"}[30s])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "irate(ceph_rbd_read_latency_sum{pool=\"$Pool\", image=\"$Image\"}[30s]) / irate(ceph_rbd_read_latency_count{pool=\"$Pool\", image=\"$Image\"}[30s])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "ns", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": true, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "label": null, + "name": "Datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "$Datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "Pool", + "options": [], + "query": "label_values(pool)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "$Datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "Image", + "options": [], + "query": "label_values(image)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "RBD Details", + "uid": "YhCYGcuZz", + "version": 7 +} \ No newline at end of file diff --git a/ceph/monitoring/prometheus/alerts/ceph_default_alerts.yml b/ceph/monitoring/prometheus/alerts/ceph_default_alerts.yml index aebd81615..acafdd6d8 100644 --- a/ceph/monitoring/prometheus/alerts/ceph_default_alerts.yml +++ b/ceph/monitoring/prometheus/alerts/ceph_default_alerts.yml @@ -47,14 +47,14 @@ groups: - name: osd rules: - alert: 10% OSDs down - expr: (sum(ceph_osd_up) / count(ceph_osd_up)) * 100 <= 90 + expr: count(ceph_osd_up == 0) / count(ceph_osd_up) * 100 >= 10 labels: severity: critical type: ceph_default oid: 1.3.6.1.4.1.50495.15.1.2.4.1 annotations: description: | - {{ $value | humanize}}% or {{with query "sum(ceph_osd_up)" }}{{ . | first | value }}{{ end }} of {{ with query "count(ceph_osd_up)"}}{{. | first | value }}{{ end }} OSDs are down (>=10%). + {{ $value | humanize }}% or {{ with query "count(ceph_osd_up == 0)" }}{{ . | first | value }}{{ end }} of {{ with query "count(ceph_osd_up)" }}{{ . | first | value }}{{ end }} OSDs are down (≥ 10%). The following OSDs are down: {{- range query "(ceph_osd_up * on(ceph_daemon) group_left(hostname) ceph_osd_metadata) == 0" }} diff --git a/ceph/qa/.teuthology_branch b/ceph/qa/.teuthology_branch new file mode 100644 index 000000000..1f7391f92 --- /dev/null +++ b/ceph/qa/.teuthology_branch @@ -0,0 +1 @@ +master diff --git a/ceph/qa/objectstore/bluestore-avl.yaml b/ceph/qa/objectstore/bluestore-hybrid.yaml similarity index 94% rename from ceph/qa/objectstore/bluestore-avl.yaml rename to ceph/qa/objectstore/bluestore-hybrid.yaml index 92cf8de48..68b9bc427 100644 --- a/ceph/qa/objectstore/bluestore-avl.yaml +++ b/ceph/qa/objectstore/bluestore-hybrid.yaml @@ -12,8 +12,8 @@ overrides: debug bluefs: 20 debug rocksdb: 10 bluestore fsck on mount: true - bluestore allocator: avl - bluefs allocator: avl + bluestore allocator: hybrid + bluefs allocator: hybrid # lower the full ratios since we can fill up a 100gb osd so quickly mon osd full ratio: .9 mon osd backfillfull_ratio: .85 diff --git a/ceph/qa/standalone/scrub/osd-scrub-repair.sh b/ceph/qa/standalone/scrub/osd-scrub-repair.sh index d5a5ee332..2008cd384 100755 --- a/ceph/qa/standalone/scrub/osd-scrub-repair.sh +++ b/ceph/qa/standalone/scrub/osd-scrub-repair.sh @@ -45,7 +45,7 @@ walk(if type == "object" then del(.mtime) else . end) | walk(if type == "object" then del(.version) else . end) | walk(if type == "object" then del(.prior_version) else . end)' -sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)' +sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print(json.dumps(ud, sort_keys=True, indent=2))' function run() { local dir=$1 diff --git a/ceph/qa/suites/fs/bugs/client_trim_caps/tasks/trim-i22073.yaml b/ceph/qa/suites/fs/bugs/client_trim_caps/tasks/trim-i22073.yaml index f0ed3366c..a86e918e6 100644 --- a/ceph/qa/suites/fs/bugs/client_trim_caps/tasks/trim-i22073.yaml +++ b/ceph/qa/suites/fs/bugs/client_trim_caps/tasks/trim-i22073.yaml @@ -17,3 +17,4 @@ tasks: - exec: client.0: - ceph_test_trim_caps + - ceph_test_ino_release_cb diff --git a/ceph/qa/suites/krbd/thrash/thrashers/mon-thrasher.yaml b/ceph/qa/suites/krbd/thrash/thrashers/mon-thrasher.yaml index 2ce44c860..415684dee 100644 --- a/ceph/qa/suites/krbd/thrash/thrashers/mon-thrasher.yaml +++ b/ceph/qa/suites/krbd/thrash/thrashers/mon-thrasher.yaml @@ -1,3 +1,7 @@ +overrides: + ceph: + log-whitelist: + - \(MON_DOWN\) tasks: - mon_thrash: revive_delay: 20 diff --git a/ceph/qa/suites/powercycle/osd/powercycle/default.yaml b/ceph/qa/suites/powercycle/osd/powercycle/default.yaml index a693f4b41..9e0ed4769 100644 --- a/ceph/qa/suites/powercycle/osd/powercycle/default.yaml +++ b/ceph/qa/suites/powercycle/osd/powercycle/default.yaml @@ -1,8 +1,24 @@ tasks: - install: extra_system_packages: - deb: ['bison', 'flex', 'libelf-dev', 'libssl-dev'] - rpm: ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel'] + deb: + - bison + - flex + - libelf-dev + - libssl-dev + - libaio-dev + - libtool-bin + - uuid-dev + - xfslibs-dev + rpm: + - bison + - flex + - elfutils-libelf-devel + - openssl-devel + - libaio-devel + - libtool + - libuuid-devel + - xfsprogs-devel - ceph: - thrashosds: chance_down: 1.0 diff --git a/ceph/qa/suites/powercycle/osd/tasks/cfuse_workunit_suites_fsx.yaml b/ceph/qa/suites/powercycle/osd/tasks/cfuse_workunit_suites_fsx.yaml index 1c2ba286b..94031518e 100644 --- a/ceph/qa/suites/powercycle/osd/tasks/cfuse_workunit_suites_fsx.yaml +++ b/ceph/qa/suites/powercycle/osd/tasks/cfuse_workunit_suites_fsx.yaml @@ -1,17 +1,5 @@ tasks: - ceph-fuse: -- install: - extra_system_packages: - deb: - - libaio-dev - - libtool-bin - - uuid-dev - - xfslibs-dev - rpm: - - libaio-devel - - libtool - - libuuid-devel - - xfsprogs-devel - workunit: timeout: 6h clients: diff --git a/ceph/qa/suites/rados/cephadm/smoke/fixed-2.yaml b/ceph/qa/suites/rados/cephadm/smoke/fixed-2.yaml index 387053022..8a3ad9f58 100644 --- a/ceph/qa/suites/rados/cephadm/smoke/fixed-2.yaml +++ b/ceph/qa/suites/rados/cephadm/smoke/fixed-2.yaml @@ -7,6 +7,7 @@ roles: - osd.2 - osd.3 - client.0 + - ceph.rgw.realm.zone.a - node-exporter.a - alertmanager.a - - mon.b diff --git a/ceph/qa/suites/rados/cephadm/upgrade/2-start-upgrade.yaml b/ceph/qa/suites/rados/cephadm/upgrade/2-start-upgrade.yaml index 6c52d78d6..f13b2e07c 100644 --- a/ceph/qa/suites/rados/cephadm/upgrade/2-start-upgrade.yaml +++ b/ceph/qa/suites/rados/cephadm/upgrade/2-start-upgrade.yaml @@ -2,4 +2,4 @@ tasks: - cephadm.shell: env: [sha1] mon.a: - - ceph orch upgrade start --image quay.io/ceph-ci/ceph:$sha1 + - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 diff --git a/ceph/qa/suites/rados/cephadm/upgrade/fixed-2.yaml b/ceph/qa/suites/rados/cephadm/upgrade/fixed-2.yaml deleted file mode 120000 index 5c3e0593c..000000000 --- a/ceph/qa/suites/rados/cephadm/upgrade/fixed-2.yaml +++ /dev/null @@ -1 +0,0 @@ -../smoke/fixed-2.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/upgrade/fixed-2.yaml b/ceph/qa/suites/rados/cephadm/upgrade/fixed-2.yaml new file mode 100644 index 000000000..8c79ebe96 --- /dev/null +++ b/ceph/qa/suites/rados/cephadm/upgrade/fixed-2.yaml @@ -0,0 +1,31 @@ +roles: +- - mon.a + - mon.c + - mgr.y + - osd.0 + - osd.1 + - osd.2 + - osd.3 + - client.0 +# - ceph.rgw.realm.zone.a # Disabled, needs 15.2.4 as an upgrade start + - node-exporter.a + - alertmanager.a +- - mon.b + - mgr.x + - osd.4 + - osd.5 + - osd.6 + - osd.7 + - client.1 + - prometheus.a + - grafana.a + - node-exporter.b +openstack: +- volumes: # attached to each instance + count: 4 + size: 10 # GB +overrides: + ceph: + conf: + osd: + osd shutdown pgref assert: true diff --git a/ceph/qa/suites/rados/cephadm/with-work/distro/ubuntu_latest.yaml b/ceph/qa/suites/rados/cephadm/with-work/distro/ubuntu_latest.yaml deleted file mode 120000 index 3a09f9abb..000000000 --- a/ceph/qa/suites/rados/cephadm/with-work/distro/ubuntu_latest.yaml +++ /dev/null @@ -1 +0,0 @@ -.qa/distros/supported/ubuntu_latest.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rados/cephadm/workunits/distro/ubuntu_latest.yaml b/ceph/qa/suites/rados/cephadm/workunits/distro/ubuntu_latest.yaml deleted file mode 120000 index 3a09f9abb..000000000 --- a/ceph/qa/suites/rados/cephadm/workunits/distro/ubuntu_latest.yaml +++ /dev/null @@ -1 +0,0 @@ -.qa/distros/supported/ubuntu_latest.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rados/dashboard/tasks/dashboard.yaml b/ceph/qa/suites/rados/dashboard/tasks/dashboard.yaml index e61294eee..f210fc1c8 100644 --- a/ceph/qa/suites/rados/dashboard/tasks/dashboard.yaml +++ b/ceph/qa/suites/rados/dashboard/tasks/dashboard.yaml @@ -52,4 +52,5 @@ tasks: - tasks.mgr.dashboard.test_role - tasks.mgr.dashboard.test_settings - tasks.mgr.dashboard.test_summary + - tasks.mgr.dashboard.test_telemetry - tasks.mgr.dashboard.test_user diff --git a/ceph/qa/suites/rgw/crypt/2-kms/barbican.yaml b/ceph/qa/suites/rgw/crypt/2-kms/barbican.yaml index a84a1627b..94c43895f 100644 --- a/ceph/qa/suites/rgw/crypt/2-kms/barbican.yaml +++ b/ceph/qa/suites/rgw/crypt/2-kms/barbican.yaml @@ -3,9 +3,21 @@ overrides: conf: client: rgw crypt s3 kms backend: barbican - rgw keystone barbican tenant: rgwcrypt + rgw keystone barbican project: rgwcrypt rgw keystone barbican user: rgwcrypt-user rgw keystone barbican password: rgwcrypt-pass + rgw keystone barbican domain: Default + rgw keystone api version: 3 + rgw keystone accepted roles: admin,Member,creator + rgw keystone implicit tenants: true + rgw keystone accepted admin roles: admin + rgw swift enforce content length: true + rgw swift account in url: true + rgw swift versioning enabled: true + rgw keystone admin project: admin + rgw keystone admin user: admin + rgw keystone admin password: ADMIN + rgw keystone admin domain: Default rgw: client.0: use-keystone-role: client.0 @@ -15,35 +27,33 @@ tasks: - tox: [ client.0 ] - keystone: client.0: - sha1: 12.0.0.0b2 + sha1: 17.0.0.0rc2 force-branch: master - tenants: - - name: admin - description: Admin Tenant + projects: - name: rgwcrypt description: Encryption Tenant + domain: default - name: barbican description: Barbican + domain: default - name: s3 description: S3 project + domain: default users: - - name: admin - password: ADMIN - project: admin - name: rgwcrypt-user password: rgwcrypt-pass project: rgwcrypt + domain: default - name: barbican-user password: barbican-pass project: barbican + domain: default - name: s3-user password: s3-pass project: s3 - roles: [ name: admin, name: Member, name: creator ] + domain: default + roles: [ name: Member, name: creator ] role-mappings: - - name: admin - user: admin - project: admin - name: Member user: rgwcrypt-user project: rgwcrypt @@ -54,9 +64,6 @@ tasks: user: s3-user project: s3 services: - - name: keystone - type: identity - description: Keystone Identity Service - name: swift type: object-store description: Swift Service diff --git a/ceph/qa/suites/rgw/crypt/4-tests/s3tests.yaml b/ceph/qa/suites/rgw/crypt/4-tests/s3tests.yaml index 9f52fee08..f713f0b95 100644 --- a/ceph/qa/suites/rgw/crypt/4-tests/s3tests.yaml +++ b/ceph/qa/suites/rgw/crypt/4-tests/s3tests.yaml @@ -1,7 +1,7 @@ tasks: - s3tests: client.0: - force-branch: ceph-master + force-branch: ceph-octopus barbican: kms_key: my-key-1 kms_key2: my-key-2 diff --git a/ceph/qa/suites/rgw/multifs/tasks/rgw_ragweed.yaml b/ceph/qa/suites/rgw/multifs/tasks/rgw_ragweed.yaml index a6ecc649a..c3d24c6c5 100644 --- a/ceph/qa/suites/rgw/multifs/tasks/rgw_ragweed.yaml +++ b/ceph/qa/suites/rgw/multifs/tasks/rgw_ragweed.yaml @@ -4,12 +4,12 @@ tasks: - rgw: [client.0] - ragweed: client.0: - default-branch: ceph-master + default-branch: ceph-octopus rgw_server: client.0 stages: prepare - ragweed: client.0: - default-branch: ceph-master + default-branch: ceph-octopus rgw_server: client.0 stages: check overrides: diff --git a/ceph/qa/suites/rgw/multifs/tasks/rgw_s3tests.yaml b/ceph/qa/suites/rgw/multifs/tasks/rgw_s3tests.yaml index 4cdded04e..d234060bd 100644 --- a/ceph/qa/suites/rgw/multifs/tasks/rgw_s3tests.yaml +++ b/ceph/qa/suites/rgw/multifs/tasks/rgw_s3tests.yaml @@ -4,7 +4,7 @@ tasks: - rgw: [client.0] - s3tests: client.0: - force-branch: ceph-master + force-branch: ceph-octopus rgw_server: client.0 overrides: ceph: diff --git a/ceph/qa/suites/rgw/tempest/tasks/rgw_tempest.yaml b/ceph/qa/suites/rgw/tempest/tasks/rgw_tempest.yaml index ad2de4858..99c776df7 100644 --- a/ceph/qa/suites/rgw/tempest/tasks/rgw_tempest.yaml +++ b/ceph/qa/suites/rgw/tempest/tasks/rgw_tempest.yaml @@ -8,24 +8,9 @@ tasks: - tox: [ client.0 ] - keystone: client.0: - sha1: 12.0.0.0b2 + sha1: 17.0.0.0rc2 force-branch: master - tenants: - - name: admin - description: Admin Tenant - users: - - name: admin - password: ADMIN - project: admin - roles: [ name: admin, name: Member ] - role-mappings: - - name: admin - user: admin - project: admin services: - - name: keystone - type: identity - description: Keystone Identity Service - name: swift type: object-store description: Swift Service @@ -35,7 +20,7 @@ tasks: use-keystone-role: client.0 - tempest: client.0: - sha1: d3fa46495a78160989120ba39793f7ba2e22d81c + sha1: d43223773d75d2e82fb33a1281038e611c41d0f3 force-branch: master use-keystone-role: client.0 auth: @@ -43,25 +28,27 @@ tasks: admin_project_name: admin admin_password: ADMIN admin_domain_name: Default + tempest_roles: admin identity: uri: http://{keystone_public_host}:{keystone_public_port}/v2.0/ uri_v3: http://{keystone_public_host}:{keystone_public_port}/v3/ + auth_version: v3 admin_role: admin + default_domain_name: Default object-storage: reseller_admin_role: admin object-storage-feature-enabled: container_sync: false - discoverability: false + discoverability: true blacklist: - # TODO(rzarzynski): we really need to update the list after - # merging PRs #15369 and #12704. Additionally, we would be - # able to enable the discoverability API testing above. - - .*test_list_containers_reverse_order.* - - .*test_list_container_contents_with_end_marker.* - - .*test_delete_non_empty_container.* + - .*test_account_quotas_negative.AccountQuotasNegativeTest.test_user_modify_quota + - .*test_container_acl_negative.ObjectACLsNegativeTest.* + - .*test_container_services_negative.ContainerNegativeTest.test_create_container_metadata_.* + - .*test_container_staticweb.StaticWebTest.test_web_index + - .*test_container_staticweb.StaticWebTest.test_web_listing_css - .*test_container_synchronization.* - - .*test_get_object_after_expiration_time.* - - .*test_create_object_with_transfer_encoding.* + - .*test_object_services.PublicObjectTest.test_access_public_container_object_without_using_creds + overrides: ceph: conf: @@ -69,10 +56,14 @@ overrides: osd_min_pg_log_entries: 10 osd_max_pg_log_entries: 10 client: - rgw keystone admin token: ADMIN + rgw keystone api version: 3 rgw keystone accepted roles: admin,Member rgw keystone implicit tenants: true rgw keystone accepted admin roles: admin rgw swift enforce content length: true rgw swift account in url: true rgw swift versioning enabled: true + rgw keystone admin domain: Default + rgw keystone admin user: admin + rgw keystone admin password: ADMIN + rgw keystone admin project: admin diff --git a/ceph/qa/suites/rgw/thrash/workload/rgw_s3tests.yaml b/ceph/qa/suites/rgw/thrash/workload/rgw_s3tests.yaml index e691920f7..5e95c9988 100644 --- a/ceph/qa/suites/rgw/thrash/workload/rgw_s3tests.yaml +++ b/ceph/qa/suites/rgw/thrash/workload/rgw_s3tests.yaml @@ -1,7 +1,7 @@ tasks: - s3tests: client.0: - force-branch: ceph-master + force-branch: ceph-octopus rgw_server: client.0 overrides: ceph: diff --git a/ceph/qa/suites/rgw/tools/+ b/ceph/qa/suites/rgw/tools/+ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/rgw/tools/.qa b/ceph/qa/suites/rgw/tools/.qa new file mode 120000 index 000000000..fea2489fd --- /dev/null +++ b/ceph/qa/suites/rgw/tools/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/ceph/qa/suites/rgw/tools/centos_latest.yaml b/ceph/qa/suites/rgw/tools/centos_latest.yaml new file mode 120000 index 000000000..bd9854e70 --- /dev/null +++ b/ceph/qa/suites/rgw/tools/centos_latest.yaml @@ -0,0 +1 @@ +.qa/distros/supported/centos_latest.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rgw/tools/cluster.yaml b/ceph/qa/suites/rgw/tools/cluster.yaml new file mode 100644 index 000000000..0eab7ebad --- /dev/null +++ b/ceph/qa/suites/rgw/tools/cluster.yaml @@ -0,0 +1,9 @@ +roles: +- [mon.a, osd.0, osd.1, osd.2, mgr.0, client.0] +openstack: +- volumes: # attached to each instance + count: 1 + size: 10 # GB +overrides: + rgw: + frontend: beast \ No newline at end of file diff --git a/ceph/qa/suites/rgw/tools/tasks.yaml b/ceph/qa/suites/rgw/tools/tasks.yaml new file mode 100644 index 000000000..acceb21c8 --- /dev/null +++ b/ceph/qa/suites/rgw/tools/tasks.yaml @@ -0,0 +1,19 @@ +tasks: +- install: +- ceph: +- rgw: + client.0: + # force rgw_dns_name to be set with the fully qualified host name; + # it will be appended to the empty string + dns-name: '' +- workunit: + clients: + client.0: + - rgw/test_rgw_orphan_list.sh +overrides: + ceph: + conf: + client: + debug rgw: 20 + debug ms: 1 + rgw enable static website: false diff --git a/ceph/qa/suites/rgw/verify/tasks/ragweed.yaml b/ceph/qa/suites/rgw/verify/tasks/ragweed.yaml index 8fbd3cefb..2788a8866 100644 --- a/ceph/qa/suites/rgw/verify/tasks/ragweed.yaml +++ b/ceph/qa/suites/rgw/verify/tasks/ragweed.yaml @@ -1,6 +1,6 @@ tasks: - ragweed: client.0: - default-branch: ceph-master + default-branch: ceph-octopus rgw_server: client.0 stages: prepare,check diff --git a/ceph/qa/suites/rgw/verify/tasks/s3tests.yaml b/ceph/qa/suites/rgw/verify/tasks/s3tests.yaml index b5aef9b04..976b7b35e 100644 --- a/ceph/qa/suites/rgw/verify/tasks/s3tests.yaml +++ b/ceph/qa/suites/rgw/verify/tasks/s3tests.yaml @@ -1,5 +1,5 @@ tasks: - s3tests: client.0: - force-branch: ceph-master + force-branch: ceph-octopus rgw_server: client.0 diff --git a/ceph/qa/suites/rgw/website/tasks/s3tests-website.yaml b/ceph/qa/suites/rgw/website/tasks/s3tests-website.yaml index 9ca6fcfbd..a0ab98f2f 100644 --- a/ceph/qa/suites/rgw/website/tasks/s3tests-website.yaml +++ b/ceph/qa/suites/rgw/website/tasks/s3tests-website.yaml @@ -12,6 +12,6 @@ tasks: dns-s3website-name: s3-website. - s3tests: client.0: - force-branch: ceph-master + force-branch: ceph-octopus rgw_server: client.0 rgw_website_server: client.1 diff --git a/ceph/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml b/ceph/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml index 29a6cec12..a029c7f9e 100644 --- a/ceph/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml +++ b/ceph/qa/suites/smoke/basic/tasks/rgw_ec_s3tests.yaml @@ -9,7 +9,7 @@ tasks: - rgw: [client.0] - s3tests: client.0: - force-branch: ceph-master + force-branch: ceph-octopus rgw_server: client.0 overrides: ceph: diff --git a/ceph/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml b/ceph/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml index 2ba6a4b49..c5528b541 100644 --- a/ceph/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml +++ b/ceph/qa/suites/smoke/basic/tasks/rgw_s3tests.yaml @@ -5,7 +5,7 @@ tasks: - rgw: [client.0] - s3tests: client.0: - force-branch: ceph-master + force-branch: ceph-octopus rgw_server: client.0 overrides: ceph: diff --git a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-parallel/point-to-point-upgrade.yaml b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-parallel/point-to-point-upgrade.yaml index 1bcca4e75..165076b01 100644 --- a/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-parallel/point-to-point-upgrade.yaml +++ b/ceph/qa/suites/upgrade/octopus-p2p/octopus-p2p-parallel/point-to-point-upgrade.yaml @@ -3,7 +3,7 @@ meta: Run ceph on two nodes, using one of them as a client, with a separate client-only node. Use xfs beneath the osds. - install ceph/octopus v15.2.1 point version + install ceph/octopus v15.2.3 and the v15.2.x point versions run workload and upgrade-sequence in parallel (every point release should be tested) run workload and upgrade-sequence in parallel @@ -83,18 +83,17 @@ tasks: - print: "**** done workload v15.2.1" -####### upgrade to v15.2.?? PLACEHOLDER -#- install.upgrade: -# #exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev'] -# mon.a: -# tag: v15.2.?? -# mon.b: -# tag: v15.2.?? -# # Note that client.a IS NOT upgraded at this point -#- parallel: -# - workload_octopus -# - upgrade-sequence_octopus -#- print: "**** done parallel octopus v15.2.??" +####### upgrade to v15.2.3 +- install.upgrade: + #exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev'] + mon.a: + tag: v15.2.3 + mon.b: + tag: v15.2.3 +- parallel: + - workload_octopus + - upgrade-sequence_octopus +- print: "**** done parallel octopus v15.2.3" #### upgrade to latest octopus - install.upgrade: diff --git a/ceph/qa/tasks/barbican.py b/ceph/qa/tasks/barbican.py index 0ce4aefb7..46b036ef4 100644 --- a/ceph/qa/tasks/barbican.py +++ b/ceph/qa/tasks/barbican.py @@ -4,8 +4,9 @@ Deploy and configure Barbican for Teuthology import argparse import contextlib import logging -import httplib -from urlparse import urlparse +import six +from six.moves import http_client +from six.moves.urllib.parse import urlparse import json from teuthology import misc as teuthology @@ -172,7 +173,7 @@ def configure_barbican(ctx, config): Configure barbican paste-api and barbican-api. """ assert isinstance(config, dict) - (cclient, cconfig) = config.items()[0] + (cclient, cconfig) = next(iter(config.items())) keystone_role = cconfig.get('use-keystone-role', None) if keystone_role is None: @@ -239,7 +240,7 @@ def create_secrets(ctx, config): Create a main and an alternate s3 user. """ assert isinstance(config, dict) - (cclient, cconfig) = config.items()[0] + (cclient, cconfig) = next(iter(config.items())) rgw_user = cconfig['rgw_user'] @@ -250,30 +251,38 @@ def create_secrets(ctx, config): port=barbican_port) log.info("barbican_url=%s", barbican_url) #fetching user_id of user that gets secrets for radosgw - token_req = httplib.HTTPConnection(keystone_host, keystone_port, timeout=30) + token_req = http_client.HTTPConnection(keystone_host, keystone_port, timeout=30) token_req.request( 'POST', - '/v2.0/tokens', + '/v3/auth/tokens', headers={'Content-Type':'application/json'}, - body=json.dumps( - {"auth": - {"passwordCredentials": - {"username": rgw_user["username"], - "password": rgw_user["password"] - }, - "tenantName": rgw_user["tenantName"] - } + body=json.dumps({ + "auth": { + "identity": { + "methods": ["password"], + "password": { + "user": { + "domain": {"id": "default"}, + "name": rgw_user["username"], + "password": rgw_user["password"] + } + } + }, + "scope": { + "project": { + "domain": {"id": "default"}, + "name": rgw_user["tenantName"] + } + } } - ) - ) + })) rgw_access_user_resp = token_req.getresponse() if not (rgw_access_user_resp.status >= 200 and rgw_access_user_resp.status < 300): raise Exception("Cannot authenticate user "+rgw_user["username"]+" for secret creation") # baru_resp = json.loads(baru_req.data) - rgw_access_user_data = json.loads(rgw_access_user_resp.read()) - rgw_user_id = rgw_access_user_data['access']['user']['id'] - + rgw_access_user_data = json.loads(six.ensure_str(rgw_access_user_resp.read())) + rgw_user_id = rgw_access_user_data['token']['user']['id'] if 'secrets' in cconfig: for secret in cconfig['secrets']: if 'name' not in secret: @@ -287,30 +296,37 @@ def create_secrets(ctx, config): if 'password' not in secret: raise ConfigError('barbican.secrets must have "password" field') - token_req = httplib.HTTPConnection(keystone_host, keystone_port, timeout=30) + token_req = http_client.HTTPConnection(keystone_host, keystone_port, timeout=30) token_req.request( 'POST', - '/v2.0/tokens', + '/v3/auth/tokens', headers={'Content-Type':'application/json'}, - body=json.dumps( - { - "auth": { - "passwordCredentials": { - "username": secret["username"], - "password": secret["password"] - }, - "tenantName":secret["tenantName"] + body=json.dumps({ + "auth": { + "identity": { + "methods": ["password"], + "password": { + "user": { + "domain": {"id": "default"}, + "name": secret["username"], + "password": secret["password"] + } + } + }, + "scope": { + "project": { + "domain": {"id": "default"}, + "name": secret["tenantName"] + } } } - ) - ) + })) token_resp = token_req.getresponse() if not (token_resp.status >= 200 and token_resp.status < 300): raise Exception("Cannot authenticate user "+secret["username"]+" for secret creation") - token_data = json.loads(token_resp.read()) - token_id = token_data['access']['token']['id'] + token_id = token_resp.getheader('x-subject-token') key1_json = json.dumps( { @@ -324,7 +340,7 @@ def create_secrets(ctx, config): "payload_content_encoding": "base64" }) - sec_req = httplib.HTTPConnection(barbican_host, barbican_port, timeout=30) + sec_req = http_client.HTTPConnection(barbican_host, barbican_port, timeout=30) try: sec_req.request( 'POST', @@ -342,7 +358,7 @@ def create_secrets(ctx, config): if not (barbican_sec_resp.status >= 200 and barbican_sec_resp.status < 300): raise Exception("Cannot create secret") - barbican_data = json.loads(barbican_sec_resp.read()) + barbican_data = json.loads(six.ensure_str(barbican_sec_resp.read())) if 'secret_ref' not in barbican_data: raise ValueError("Malformed secret creation response") secret_ref = barbican_data["secret_ref"] @@ -355,7 +371,7 @@ def create_secrets(ctx, config): "project-access": True } }) - acl_req = httplib.HTTPConnection(secret_url_parsed.netloc, timeout=30) + acl_req = http_client.HTTPConnection(secret_url_parsed.netloc, timeout=30) acl_req.request( 'PUT', secret_url_parsed.path+'/acl', @@ -393,11 +409,9 @@ def task(ctx, config): - tox: [ client.0 ] - keystone: client.0: - sha1: 12.0.0.0b2 + sha1: 17.0.0.0rc2 force-branch: master - tenants: - - name: admin - description: Admin Tenant + projects: - name: rgwcrypt description: Encryption Tenant - name: barbican @@ -405,9 +419,6 @@ def task(ctx, config): - name: s3 description: S3 project users: - - name: admin - password: ADMIN - project: admin - name: rgwcrypt-user password: rgwcrypt-pass project: rgwcrypt @@ -417,11 +428,8 @@ def task(ctx, config): - name: s3-user password: s3-pass project: s3 - roles: [ name: admin, name: Member, name: creator ] + roles: [ name: Member, name: creator ] role-mappings: - - name: admin - user: admin - project: admin - name: Member user: rgwcrypt-user project: rgwcrypt diff --git a/ceph/qa/tasks/cbt.py b/ceph/qa/tasks/cbt.py index c5567a05d..3255107ad 100644 --- a/ceph/qa/tasks/cbt.py +++ b/ceph/qa/tasks/cbt.py @@ -44,7 +44,7 @@ class CBT(Task): ) benchmark_config = self.config.get('benchmarks') - benchmark_type = benchmark_config.keys()[0] + benchmark_type = next(iter(benchmark_config.keys())) if benchmark_type in ['librbdfio', 'fio']: testdir = misc.get_testdir(self.ctx) benchmark_config[benchmark_type]['cmd_path'] = os.path.join(testdir, 'fio/fio') @@ -80,7 +80,7 @@ class CBT(Task): cbt_depends = ['python3-yaml', 'python3-lxml', 'librbd-dev', 'collectl'] self.first_mon.run(args=install_cmd + cbt_depends) - benchmark_type = self.cbt_config.get('benchmarks').keys()[0] + benchmark_type = next(iter(self.cbt_config.get('benchmarks').keys())) self.log.info('benchmark: %s', benchmark_type) if benchmark_type in ['librbdfio', 'fio']: @@ -201,7 +201,7 @@ class CBT(Task): def setup(self): super(CBT, self).setup() - self.first_mon = self.ctx.cluster.only(misc.get_first_mon(self.ctx, self.config)).remotes.keys()[0] + self.first_mon = next(iter(self.ctx.cluster.only(misc.get_first_mon(self.ctx, self.config)).remotes.keys())) self.cbt_config = self.generate_cbt_config() self.log.info('cbt configuration is %s', self.cbt_config) self.cbt_dir = os.path.join(misc.get_archive_dir(self.ctx), 'cbt') @@ -233,7 +233,7 @@ class CBT(Task): '{tdir}/cbt'.format(tdir=testdir), ] ) - benchmark_type = self.cbt_config.get('benchmarks').keys()[0] + benchmark_type = next(iter(self.cbt_config.get('benchmarks').keys())) if benchmark_type in ['librbdfio', 'fio']: self.first_mon.run( args=[ diff --git a/ceph/qa/tasks/ceph_fuse.py b/ceph/qa/tasks/ceph_fuse.py index 4f06e2b76..1439ccffd 100644 --- a/ceph/qa/tasks/ceph_fuse.py +++ b/ceph/qa/tasks/ceph_fuse.py @@ -6,7 +6,7 @@ import contextlib import logging from teuthology import misc as teuthology -from cephfs.fuse_mount import FuseMount +from tasks.cephfs.fuse_mount import FuseMount log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/ceph_manager.py b/ceph/qa/tasks/ceph_manager.py index 5742cff75..c058735aa 100644 --- a/ceph/qa/tasks/ceph_manager.py +++ b/ceph/qa/tasks/ceph_manager.py @@ -16,6 +16,7 @@ import os import six from io import BytesIO +from six import StringIO from teuthology import misc as teuthology from tasks.scrub import Scrubber from tasks.util.rados import cmd_erasure_code_profile @@ -1239,8 +1240,8 @@ class ObjectStoreTool: self.pgid = self.manager.get_object_pg_with_shard(self.pool, self.object_name, self.osd) - self.remote = self.manager.ctx.\ - cluster.only('osd.{o}'.format(o=self.osd)).remotes.keys()[0] + self.remote = next(iter(self.manager.ctx.\ + cluster.only('osd.{o}'.format(o=self.osd)).remotes.keys())) path = self.manager.get_filepath().format(id=self.osd) self.paths = ("--data-path {path} --journal-path {path}/journal". format(path=path)) @@ -1409,7 +1410,7 @@ class CephManager: if watch_channel is not None: args.append("--watch-channel") args.append(watch_channel) - return self.controller.run(args=args, wait=False, stdout=BytesIO(), stdin=run.PIPE) + return self.controller.run(args=args, wait=False, stdout=StringIO(), stdin=run.PIPE) def get_mon_socks(self): """ @@ -1592,7 +1593,7 @@ class CephManager: def osd_admin_socket(self, osd_id, command, check_status=True, timeout=0, stdout=None): if stdout is None: - stdout = BytesIO() + stdout = StringIO() return self.admin_socket('osd', osd_id, command, check_status, timeout, stdout) def find_remote(self, service_type, service_id): @@ -1616,7 +1617,7 @@ class CephManager: to the admin socket """ if stdout is None: - stdout = BytesIO() + stdout = StringIO() remote = self.find_remote(service_type, service_id) @@ -1745,7 +1746,7 @@ class CephManager: five seconds and try again. """ if stdout is None: - stdout = BytesIO() + stdout = StringIO() tries = 0 while True: proc = self.admin_socket(service_type, service_id, @@ -1819,9 +1820,9 @@ class CephManager: """ Get osd statuses sorted by states that the osds are in. """ - osd_lines = filter( + osd_lines = list(filter( lambda x: x.startswith('osd.') and (("up" in x) or ("down" in x)), - self.raw_osd_status().split('\n')) + self.raw_osd_status().split('\n'))) self.log(osd_lines) in_osds = [int(i[4:].split()[0]) for i in filter(lambda x: " in " in x, osd_lines)] @@ -1965,7 +1966,7 @@ class CephManager: """ with self.lock: if self.pools: - return random.choice(self.pools.keys()) + return random.sample(self.pools.keys(), 1)[0] def get_pool_pg_num(self, pool_name): """ diff --git a/ceph/qa/tasks/ceph_objectstore_tool.py b/ceph/qa/tasks/ceph_objectstore_tool.py index 641981ebd..d4facfd78 100644 --- a/ceph/qa/tasks/ceph_objectstore_tool.py +++ b/ceph/qa/tasks/ceph_objectstore_tool.py @@ -4,21 +4,20 @@ ceph_objectstore_tool - Simple test of ceph-objectstore-tool utility from io import BytesIO import contextlib +import json import logging -import ceph_manager -from teuthology import misc as teuthology -import time import os import six -import string -from teuthology.orchestra import run import sys import tempfile -import json +import time +from tasks import ceph_manager +from tasks.util.rados import (rados, create_replicated_pool, create_ec_pool) +from teuthology import misc as teuthology +from teuthology.orchestra import run from teuthology.exceptions import CommandFailedError -from util.rados import (rados, create_replicated_pool, create_ec_pool) # from util.rados import (rados, create_ec_pool, # create_replicated_pool, # create_cache_pool) @@ -242,7 +241,7 @@ def test_objectstore(ctx, config, cli_remote, REP_POOL, REP_NAME, ec=False): REP_NAME, DATALINECOUNT) allremote = [] allremote.append(cli_remote) - allremote += osds.remotes.keys() + allremote += list(osds.remotes.keys()) allremote = list(set(allremote)) for remote in allremote: cod_setup_remote_data(log, ctx, remote, NUM_OBJECTS, DATADIR, @@ -288,7 +287,7 @@ def test_objectstore(ctx, config, cli_remote, REP_POOL, REP_NAME, ec=False): log.debug(remote) log.debug(osds.remotes[remote]) for role in osds.remotes[remote]: - if string.find(role, "osd.") != 0: + if not role.startswith("osd."): continue osdid = int(role.split('.')[1]) log.info("process osd.{id} on {remote}". @@ -325,7 +324,7 @@ def test_objectstore(ctx, config, cli_remote, REP_POOL, REP_NAME, ec=False): for remote in osds.remotes.keys(): for role in osds.remotes[remote]: - if string.find(role, "osd.") != 0: + if not role.startswith("osd."): continue osdid = int(role.split('.')[1]) if osdid not in pgs: @@ -415,7 +414,7 @@ def test_objectstore(ctx, config, cli_remote, REP_POOL, REP_NAME, ec=False): for remote in osds.remotes.keys(): for role in osds.remotes[remote]: - if string.find(role, "osd.") != 0: + if not role.startswith("osd."): continue osdid = int(role.split('.')[1]) if osdid not in pgs: @@ -500,7 +499,7 @@ def test_objectstore(ctx, config, cli_remote, REP_POOL, REP_NAME, ec=False): log.info("Test pg info") for remote in osds.remotes.keys(): for role in osds.remotes[remote]: - if string.find(role, "osd.") != 0: + if not role.startswith("osd."): continue osdid = int(role.split('.')[1]) if osdid not in pgs: @@ -523,7 +522,7 @@ def test_objectstore(ctx, config, cli_remote, REP_POOL, REP_NAME, ec=False): log.info("Test pg logging") for remote in osds.remotes.keys(): for role in osds.remotes[remote]: - if string.find(role, "osd.") != 0: + if not role.startswith("osd."): continue osdid = int(role.split('.')[1]) if osdid not in pgs: @@ -554,7 +553,7 @@ def test_objectstore(ctx, config, cli_remote, REP_POOL, REP_NAME, ec=False): EXP_ERRORS = 0 for remote in osds.remotes.keys(): for role in osds.remotes[remote]: - if string.find(role, "osd.") != 0: + if not role.startswith("osd."): continue osdid = int(role.split('.')[1]) if osdid not in pgs: @@ -580,7 +579,7 @@ def test_objectstore(ctx, config, cli_remote, REP_POOL, REP_NAME, ec=False): RM_ERRORS = 0 for remote in osds.remotes.keys(): for role in osds.remotes[remote]: - if string.find(role, "osd.") != 0: + if not role.startswith("osd."): continue osdid = int(role.split('.')[1]) if osdid not in pgs: @@ -605,7 +604,7 @@ def test_objectstore(ctx, config, cli_remote, REP_POOL, REP_NAME, ec=False): for remote in osds.remotes.keys(): for role in osds.remotes[remote]: - if string.find(role, "osd.") != 0: + if not role.startswith("osd."): continue osdid = int(role.split('.')[1]) if osdid not in pgs: diff --git a/ceph/qa/tasks/cephadm.py b/ceph/qa/tasks/cephadm.py index 2c64ca997..a5065ba3e 100644 --- a/ceph/qa/tasks/cephadm.py +++ b/ceph/qa/tasks/cephadm.py @@ -1,19 +1,22 @@ """ Ceph cluster task, deployed via cephadm orchestrator """ -from io import BytesIO - import argparse import configobj import contextlib +import errno import logging import os import json import re import uuid -from ceph_manager import CephManager +import six +import toml +from io import BytesIO +from six import StringIO from tarfile import ReadError +from tasks.ceph_manager import CephManager from teuthology import misc as teuthology from teuthology import contextutil from teuthology.orchestra import run @@ -189,7 +192,7 @@ def ceph_log(ctx, config): run.Raw('|'), 'head', '-n', '1', ]) r = ctx.ceph[cluster_name].bootstrap_remote.run( - stdout=BytesIO(), + stdout=StringIO(), args=args, ) stdout = r.stdout.getvalue() @@ -250,8 +253,11 @@ def ceph_log(ctx, config): os.makedirs(sub) except OSError: pass - teuthology.pull_directory(remote, '/var/log/ceph', # everything - os.path.join(sub, 'log')) + try: + teuthology.pull_directory(remote, '/var/log/ceph', # everything + os.path.join(sub, 'log')) + except ReadError: + pass @contextlib.contextmanager def ceph_crash(ctx, config): @@ -286,7 +292,15 @@ def ceph_crash(ctx, config): pass @contextlib.contextmanager -def ceph_bootstrap(ctx, config): +def ceph_bootstrap(ctx, config, registry): + """ + Bootstrap ceph cluster, setup containers' registry mirror before + the bootstrap if the registry is provided. + + :param ctx: the argparse.Namespace object + :param config: the config dict + :param registry: url to containers' mirror registry + """ cluster_name = config['cluster'] testdir = teuthology.get_testdir(ctx) fsid = ctx.ceph[cluster_name].fsid @@ -302,6 +316,8 @@ def ceph_bootstrap(ctx, config): ctx.cluster.run(args=[ 'sudo', 'chmod', '777', '/etc/ceph', ]); + if registry: + add_mirror_to_cluster(ctx, registry) try: # write seed config log.info('Writing seed config...') @@ -312,7 +328,7 @@ def ceph_bootstrap(ctx, config): remote=bootstrap_remote, path='{}/seed.{}.conf'.format(testdir, cluster_name), data=conf_fp.getvalue()) - log.debug('Final config:\n' + conf_fp.getvalue()) + log.debug('Final config:\n' + conf_fp.getvalue().decode()) ctx.ceph[cluster_name].conf = seed_config # register initial daemons @@ -391,7 +407,7 @@ def ceph_bootstrap(ctx, config): ssh_pub_key = teuthology.get_file( remote=bootstrap_remote, path='{}/{}.pub'.format(testdir, cluster_name) - ).strip() + ).decode('ascii').strip() log.info('Installing pub ssh key for root users...') ctx.cluster.run(args=[ @@ -429,7 +445,7 @@ def ceph_bootstrap(ctx, config): ]) r = _shell(ctx, cluster_name, remote, ['ceph', 'orch', 'host', 'ls', '--format=json'], - stdout=BytesIO()) + stdout=StringIO()) hosts = [node['hostname'] for node in json.loads(r.stdout.getvalue())] assert remote.shortname in hosts @@ -449,9 +465,13 @@ def ceph_bootstrap(ctx, config): #ctx.cluster.run(args=['sudo', 'systemctl', 'stop', 'ceph.target']) # so, stop them individually - for role in ctx.daemons.resolve_role_list(None, CEPH_ROLE_TYPES): + for role in ctx.daemons.resolve_role_list(None, CEPH_ROLE_TYPES, True): cluster, type_, id_ = teuthology.split_role(role) - ctx.daemons.get_daemon(type_, id_, cluster).stop() + try: + ctx.daemons.get_daemon(type_, id_, cluster).stop() + except Exception: + log.exception('Failed to stop "{role}"'.format(role=role)) + raise # clean up /etc/ceph ctx.cluster.run(args=[ @@ -501,7 +521,7 @@ def ceph_mons(ctx, config): args=[ 'ceph', 'mon', 'dump', '-f', 'json', ], - stdout=BytesIO(), + stdout=StringIO(), ) j = json.loads(r.stdout.getvalue()) if len(j['mons']) == num_mons: @@ -516,7 +536,7 @@ def ceph_mons(ctx, config): args=[ 'ceph', 'config', 'generate-minimal-conf', ], - stdout=BytesIO(), + stdout=StringIO(), ) ctx.ceph[cluster_name].config_file = r.stdout.getvalue() @@ -595,7 +615,10 @@ def ceph_osds(ctx, config): devs = devs_by_remote[remote] assert devs ## FIXME ## dev = devs.pop() - short_dev = dev.replace('/dev/', '') + if all(_ in dev for _ in ('lv', 'vg')): + short_dev = dev.replace('/dev/', '') + else: + short_dev = dev log.info('Deploying %s on %s with %s...' % ( osd, remote.shortname, dev)) _shell(ctx, cluster_name, remote, [ @@ -709,12 +732,27 @@ def ceph_rgw(ctx, config): nodes[realmzone] = [] nodes[realmzone].append(remote.shortname + '=' + id_) daemons[role] = (remote, id_) + + for realmzone in nodes.keys(): + (realm, zone) = realmzone.split('.', 1) + + # TODO: those should be moved to mgr/cephadm + _shell(ctx, cluster_name, remote, + ['radosgw-admin', 'realm', 'create', '--rgw-realm', realm, '--default'] + ) + _shell(ctx, cluster_name, remote, + ['radosgw-admin', 'zonegroup', 'create', '--rgw-zonegroup=default', '--master', '--default'] + ) + _shell(ctx, cluster_name, remote, + ['radosgw-admin', 'zone', 'create', '--rgw-zonegroup=default', '--rgw-zone', zone, '--master', '--default'] + ) + for realmzone, nodes in nodes.items(): (realm, zone) = realmzone.split('.', 1) _shell(ctx, cluster_name, remote, [ - 'ceph', 'orch', 'apply', 'rgw', - realm, zone, - str(len(nodes)) + ';' + ';'.join(nodes)] + 'ceph', 'orch', 'apply', 'rgw', realm, zone, + '--placement', + str(len(nodes)) + ';' + ';'.join(nodes)] ) for role, i in daemons.items(): remote, id_ = i @@ -756,7 +794,7 @@ def ceph_clients(ctx, config): 'mds', 'allow *', 'mgr', 'allow *', ], - stdout=BytesIO(), + stdout=StringIO(), ) keyring = r.stdout.getvalue() teuthology.sudo_write_file( @@ -958,36 +996,23 @@ def crush_setup(ctx, config): yield @contextlib.contextmanager -def task(ctx, config): - if config is None: - config = {} - - assert isinstance(config, dict), \ - "task only supports a dictionary for configuration" - - overrides = ctx.config.get('overrides', {}) - teuthology.deep_merge(config, overrides.get('ceph', {})) - log.info('Config: ' + str(config)) - - testdir = teuthology.get_testdir(ctx) +def _bypass(): + yield - # set up cluster context - first_ceph_cluster = False - if not hasattr(ctx, 'daemons'): - first_ceph_cluster = True - if not hasattr(ctx, 'ceph'): - ctx.ceph = {} - ctx.managers = {} - if 'cluster' not in config: - config['cluster'] = 'ceph' +@contextlib.contextmanager +def initialize_config(ctx, config): cluster_name = config['cluster'] - ctx.ceph[cluster_name] = argparse.Namespace() + testdir = teuthology.get_testdir(ctx) ctx.ceph[cluster_name].thrashers = [] # fixme: setup watchdog, ala ceph.py ctx.ceph[cluster_name].roleless = False # see below + first_ceph_cluster = False + if not hasattr(ctx, 'daemons'): + first_ceph_cluster = True + # cephadm mode? if 'cephadm_mode' not in config: config['cephadm_mode'] = 'root' @@ -1002,21 +1027,6 @@ def task(ctx, config): ctx.daemons = DaemonGroup( use_cephadm=ctx.cephadm) - # image - ctx.ceph[cluster_name].image = config.get('image') - ref = None - if not ctx.ceph[cluster_name].image: - sha1 = config.get('sha1') - if sha1: - ctx.ceph[cluster_name].image = 'quay.io/ceph-ci/ceph:%s' % sha1 - ref = sha1 - else: - # hmm, fall back to branch? - branch = config.get('branch', 'master') - ref = branch - ctx.ceph[cluster_name].image = 'quay.io/ceph-ci/ceph:%s' % branch - log.info('Cluster image is %s' % ctx.ceph[cluster_name].image) - # uuid fsid = str(uuid.uuid1()) log.info('Cluster fsid is %s' % fsid) @@ -1025,22 +1035,21 @@ def task(ctx, config): # mon ips log.info('Choosing monitor IPs and ports...') remotes_and_roles = ctx.cluster.remotes.items() - roles = [role_list for (remote, role_list) in remotes_and_roles] ips = [host for (host, port) in (remote.ssh.get_transport().getpeername() for (remote, role_list) in remotes_and_roles)] if config.get('roleless', False): # mons will be named after hosts - n = len(roles) - roles = [] first_mon = None for remote, _ in remotes_and_roles: - roles.append(['mon.' + remote.shortname]) + ctx.cluster.remotes[remote].append('mon.' + remote.shortname) if not first_mon: first_mon = remote.shortname bootstrap_remote = remote - log.info('No roles; fabricating mons %s' % roles) - + log.info('No mon roles; fabricating mons') + + roles = [role_list for (remote, role_list) in ctx.cluster.remotes.items()] + ctx.ceph[cluster_name].mons = get_mons( roles, ips, cluster_name, mon_bind_msgr2=config.get('mon_bind_msgr2', True), @@ -1071,15 +1080,112 @@ def task(ctx, config): _, _, first_mgr = teuthology.split_role(mgrs[0]) log.info('First mgr is %s' % (first_mgr)) ctx.ceph[cluster_name].first_mgr = first_mgr - + yield + +@contextlib.contextmanager +def task(ctx, config): + """ + Deploy ceph cluster using cephadm + + Setup containers' mirrors before the bootstrap, if corresponding + config provided in teuthology server config yaml file. + + For example, teuthology.yaml can contain the 'defaults' section: + + defaults: + cephadm: + containers: + registry_mirrors: + docker.io: 'registry.mirror.example.com:5000' + image: 'quay.io/ceph-ci/ceph' + + Using overrides makes it possible to customize it per run. + The equivalent 'overrides' section looks like: + + overrides: + cephadm: + containers: + registry_mirrors: + docker.io: 'registry.mirror.example.com:5000' + image: 'quay.io/ceph-ci/ceph' + + :param ctx: the argparse.Namespace object + :param config: the config dict + """ + if config is None: + config = {} + + assert isinstance(config, dict), \ + "task only supports a dictionary for configuration" + + overrides = ctx.config.get('overrides', {}) + teuthology.deep_merge(config, overrides.get('ceph', {})) + teuthology.deep_merge(config, overrides.get('cephadm', {})) + log.info('Config: ' + str(config)) + + testdir = teuthology.get_testdir(ctx) + + # set up cluster context + if not hasattr(ctx, 'ceph'): + ctx.ceph = {} + ctx.managers = {} + if 'cluster' not in config: + config['cluster'] = 'ceph' + cluster_name = config['cluster'] + if cluster_name not in ctx.ceph: + ctx.ceph[cluster_name] = argparse.Namespace() + ctx.ceph[cluster_name].bootstrapped = False + + # image + teuth_defaults = teuth_config.get('defaults', {}) + cephadm_defaults = teuth_defaults.get('cephadm', {}) + containers_defaults = cephadm_defaults.get('containers', {}) + mirrors_defaults = containers_defaults.get('registry_mirrors', {}) + container_registry_mirror = mirrors_defaults.get('docker.io', None) + container_image_name = containers_defaults.get('image', None) + + containers = config.get('containers', {}) + mirrors = containers.get('registry_mirrors', {}) + container_image_name = containers.get('image', container_image_name) + container_registry_mirror = mirrors.get('docker.io', + container_registry_mirror) + + if not container_image_name: + raise Exception("Configuration error occurred. " + "The 'image' value is undefined for 'cephadm' task. " + "Please provide corresponding options in the task's " + "config, task 'overrides', or teuthology 'defaults' " + "section.") + + if not hasattr(ctx.ceph[cluster_name], 'image'): + ctx.ceph[cluster_name].image = config.get('image') + ref = None + if not ctx.ceph[cluster_name].image: + sha1 = config.get('sha1') + if sha1: + ctx.ceph[cluster_name].image = container_image_name + ':' + sha1 + ref = sha1 + else: + # hmm, fall back to branch? + branch = config.get('branch', 'master') + ref = branch + ctx.ceph[cluster_name].image = container_image_name + ':' + branch + log.info('Cluster image is %s' % ctx.ceph[cluster_name].image) + with contextutil.nested( + #if the cluster is already bootstrapped bypass corresponding methods + lambda: _bypass() if (ctx.ceph[cluster_name].bootstrapped)\ + else initialize_config(ctx=ctx, config=config), lambda: ceph_initial(), lambda: normalize_hostnames(ctx=ctx), - lambda: download_cephadm(ctx=ctx, config=config, ref=ref), + lambda: _bypass() if (ctx.ceph[cluster_name].bootstrapped)\ + else download_cephadm(ctx=ctx, config=config, ref=ref), lambda: ceph_log(ctx=ctx, config=config), lambda: ceph_crash(ctx=ctx, config=config), - lambda: ceph_bootstrap(ctx=ctx, config=config), + lambda: _bypass() if (ctx.ceph[cluster_name].bootstrapped)\ + else ceph_bootstrap(ctx, config, + container_registry_mirror), lambda: crush_setup(ctx=ctx, config=config), lambda: ceph_mons(ctx=ctx, config=config), lambda: distribute_config_and_admin_keyring(ctx=ctx, config=config), @@ -1110,3 +1216,59 @@ def task(ctx, config): finally: log.info('Teardown begin') + + +def registries_add_mirror_to_docker_io(conf, mirror): + config = toml.loads(conf) + is_v1 = 'registries' in config + if is_v1: + search = config.get('registries', {}).get('search', {}).get('registries', []) + insecure = config.get('registries', {}).get('search', {}).get('insecure', []) + # v2: MutableMapping[str, Any] = { needs Python 3 + v2 = { + 'unqualified-search-registries': search, + 'registry': [ + { + 'prefix': reg, + 'location': reg, + 'insecure': reg in insecure, + 'blocked': False, + } for reg in search + ] + } + else: + v2 = config # type: ignore + dockers = [r for r in v2['registry'] if r['prefix'] == 'docker.io'] + if dockers: + docker = dockers[0] + docker['mirror'] = [{ + "location": mirror, + "insecure": True, + }] + return v2 + + +def add_mirror_to_cluster(ctx, mirror): + log.info('Adding local image mirror %s' % mirror) + + registries_conf = '/etc/containers/registries.conf' + + for remote in ctx.cluster.remotes.keys(): + try: + config = teuthology.get_file( + remote=remote, + path=registries_conf + ) + new_config = toml.dumps(registries_add_mirror_to_docker_io(config.decode('utf-8'), mirror)) + + teuthology.sudo_write_file( + remote=remote, + path=registries_conf, + data=six.ensure_str(new_config), + ) + except IOError as e: # py3: use FileNotFoundError instead. + if e.errno != errno.ENOENT: + raise + + # Docker doesn't ship a registries.conf + log.info('Failed to add mirror: %s' % str(e)) diff --git a/ceph/qa/tasks/cephfs/cephfs_test_case.py b/ceph/qa/tasks/cephfs/cephfs_test_case.py index d9464dd41..e69941dfd 100644 --- a/ceph/qa/tasks/cephfs/cephfs_test_case.py +++ b/ceph/qa/tasks/cephfs/cephfs_test_case.py @@ -4,12 +4,12 @@ import logging from tasks.ceph_test_case import CephTestCase import os import re -from StringIO import StringIO from tasks.cephfs.fuse_mount import FuseMount from teuthology.orchestra import run from teuthology.orchestra.run import CommandFailedError +from teuthology.contextutil import safe_while log = logging.getLogger(__name__) @@ -143,8 +143,7 @@ class CephFSTestCase(CephTestCase): # Mount the requested number of clients for i in range(0, self.CLIENTS_REQUIRED): - self.mounts[i].mount() - self.mounts[i].wait_until_mounted() + self.mounts[i].mount_wait() if self.REQUIRE_RECOVERY_FILESYSTEM: if not self.REQUIRE_FILESYSTEM: @@ -163,7 +162,7 @@ class CephFSTestCase(CephTestCase): # Load an config settings of interest for setting in self.LOAD_SETTINGS: setattr(self, setting, float(self.fs.mds_asok( - ['config', 'get', setting], self.mds_cluster.mds_ids[0] + ['config', 'get', setting], list(self.mds_cluster.mds_ids)[0] )[setting])) self.configs_set = set() @@ -252,7 +251,7 @@ class CephFSTestCase(CephTestCase): timeout=30 ) except RuntimeError: - log.warn("Timeout waiting for daemons {0}, while we have {1}".format( + log.warning("Timeout waiting for daemons {0}, while we have {1}".format( daemon_ids, get_daemon_names() )) raise @@ -260,21 +259,21 @@ class CephFSTestCase(CephTestCase): def delete_mds_coredump(self, daemon_id): # delete coredump file, otherwise teuthology.internal.coredump will # catch it later and treat it as a failure. - p = self.mds_cluster.mds_daemons[daemon_id].remote.run(args=[ - "sudo", "sysctl", "-n", "kernel.core_pattern"], stdout=StringIO()) - core_dir = os.path.dirname(p.stdout.getvalue().strip()) + core_pattern = self.mds_cluster.mds_daemons[daemon_id].remote.sh( + "sudo sysctl -n kernel.core_pattern") + core_dir = os.path.dirname(core_pattern.strip()) if core_dir: # Non-default core_pattern with a directory in it # We have seen a core_pattern that looks like it's from teuthology's coredump # task, so proceed to clear out the core file log.info("Clearing core from directory: {0}".format(core_dir)) # Verify that we see the expected single coredump - ls_proc = self.mds_cluster.mds_daemons[daemon_id].remote.run(args=[ + ls_output = self.mds_cluster.mds_daemons[daemon_id].remote.sh([ "cd", core_dir, run.Raw('&&'), "sudo", "ls", run.Raw('|'), "sudo", "xargs", "file" - ], stdout=StringIO()) + ]) cores = [l.partition(":")[0] - for l in ls_proc.stdout.getvalue().strip().split("\n") + for l in ls_output.strip().split("\n") if re.match(r'.*ceph-mds.* -i +{0}'.format(daemon_id), l)] log.info("Enumerated cores: {0}".format(cores)) @@ -292,7 +291,7 @@ class CephFSTestCase(CephTestCase): timeout = 30 pause = 2 test = sorted(test) - for i in range(timeout/pause): + for i in range(timeout // pause): subtrees = self.fs.mds_asok(["get", "subtrees"], mds_id=status.get_rank(self.fs.id, rank)['name']) subtrees = filter(lambda s: s['dir']['path'].startswith('/'), subtrees) filtered = sorted([(s['dir']['path'], s['auth_first']) for s in subtrees]) @@ -304,3 +303,12 @@ class CephFSTestCase(CephTestCase): return subtrees time.sleep(pause) raise RuntimeError("rank {0} failed to reach desired subtree state".format(rank)) + + def _wait_until_scrub_complete(self, path="/", recursive=True): + out_json = self.fs.rank_tell(["scrub", "start", path] + ["recursive"] if recursive else []) + with safe_while(sleep=10, tries=10) as proceed: + while proceed(): + out_json = self.fs.rank_tell(["scrub", "status"]) + if out_json['status'] == "no active scrubs running": + break; + diff --git a/ceph/qa/tasks/cephfs/filesystem.py b/ceph/qa/tasks/cephfs/filesystem.py index 7352f12c8..2210fa970 100644 --- a/ceph/qa/tasks/cephfs/filesystem.py +++ b/ceph/qa/tasks/cephfs/filesystem.py @@ -10,6 +10,9 @@ import errno import random import traceback +from io import BytesIO +from six import StringIO + from teuthology.exceptions import CommandFailedError from teuthology import misc from teuthology.nuke import clear_firewall @@ -160,7 +163,7 @@ class FSStatus(object): if info: return info['addr'] else: - log.warn(json.dumps(list(self.get_all()), indent=2)) # dump for debugging + log.warning(json.dumps(list(self.get_all()), indent=2)) # dump for debugging raise RuntimeError("MDS id '{0}' not found in map".format(name)) def get_mds_gid(self, gid): @@ -282,7 +285,8 @@ class MDSCluster(CephCluster): return super(MDSCluster, self).get_config(key, service_type) # Some tests stop MDS daemons, don't send commands to a dead one: - service_id = random.sample(filter(lambda i: self.mds_daemons[i].running(), self.mds_daemons), 1)[0] + running_daemons = [i for i, mds in self.mds_daemons.items() if mds.running()] + service_id = random.sample(running_daemons, 1)[0] return self.json_asok(['config', 'get', key], service_type, service_id)[key] def mds_stop(self, mds_id=None): @@ -691,7 +695,7 @@ class Filesystem(MDSCluster): if refresh or self.data_pools is None: self.get_pool_names(refresh = True) assert(len(self.data_pools) == 1) - return self.data_pools.values()[0] + return next(iter(self.data_pools.values())) def get_data_pool_id(self, refresh = False): """ @@ -701,12 +705,12 @@ class Filesystem(MDSCluster): if refresh or self.data_pools is None: self.get_pool_names(refresh = True) assert(len(self.data_pools) == 1) - return self.data_pools.keys()[0] + return next(iter(self.data_pools.keys())) def get_data_pool_names(self, refresh = False): if refresh or self.data_pools is None: self.get_pool_names(refresh = True) - return self.data_pools.values() + return list(self.data_pools.values()) def get_metadata_pool_name(self): return self.metadata_pool_name @@ -1090,11 +1094,12 @@ class Filesystem(MDSCluster): os.path.join(self._prefix, "rados"), "-p", pool, "getxattr", obj_name, xattr_name ] try: - data = remote.sh(args) + proc = remote.run(args=args, stdout=BytesIO()) except CommandFailedError as e: log.error(e.__str__()) raise ObjectNotFound(obj_name) + data = proc.stdout.getvalue() dump = remote.sh( [os.path.join(self._prefix, "ceph-dencoder"), "type", type, @@ -1180,7 +1185,7 @@ class Filesystem(MDSCluster): want_objects = [ "{0:x}.{1:08x}".format(ino, n) - for n in range(0, ((size - 1) / stripe_size) + 1) + for n in range(0, ((size - 1) // stripe_size) + 1) ] exist_objects = self.rados(["ls"], pool=self.get_data_pool_name()).split("\n") @@ -1226,7 +1231,8 @@ class Filesystem(MDSCluster): return True def rados(self, args, pool=None, namespace=None, stdin_data=None, - stdin_file=None): + stdin_file=None, + stdout_data=None): """ Call into the `rados` CLI from an MDS """ @@ -1247,9 +1253,13 @@ class Filesystem(MDSCluster): if stdin_file is not None: args = ["bash", "-c", "cat " + stdin_file + " | " + " ".join(args)] + if stdout_data is None: + stdout_data = StringIO() - output = remote.sh(args, stdin=stdin_data) - return output.strip() + p = remote.run(args=args, + stdin=stdin_data, + stdout=stdout_data) + return p.stdout.getvalue().strip() def list_dirfrag(self, dir_ino): """ @@ -1326,7 +1336,7 @@ class Filesystem(MDSCluster): base_args.extend(["--rank", "%s" % str(rank)]) t1 = datetime.datetime.now() - r = self.tool_remote.sh(base_args + args).strip() + r = self.tool_remote.sh(script=base_args + args, stdout=StringIO()).strip() duration = datetime.datetime.now() - t1 log.info("Ran {0} in time {1}, result:\n{2}".format( base_args + args, duration, r diff --git a/ceph/qa/tasks/cephfs/fuse_mount.py b/ceph/qa/tasks/cephfs/fuse_mount.py index 6ca08a460..57c59acab 100644 --- a/ceph/qa/tasks/cephfs/fuse_mount.py +++ b/ceph/qa/tasks/cephfs/fuse_mount.py @@ -1,14 +1,18 @@ -from StringIO import StringIO +from io import BytesIO +from io import StringIO import json import time import logging + +import six + from textwrap import dedent from teuthology import misc from teuthology.contextutil import MaxWhileTries from teuthology.orchestra import run from teuthology.orchestra.run import CommandFailedError -from .mount import CephFSMount +from tasks.cephfs.mount import CephFSMount log = logging.getLogger(__name__) @@ -36,7 +40,7 @@ class FuseMount(CephFSMount): # failures) and ensure the mount is not left half-up. # Otherwise we might leave a zombie mount point that causes # anyone traversing cephtest/ to get hung up on. - log.warn("Trying to clean up after failed mount") + log.warning("Trying to clean up after failed mount") self.umount_wait(force=True) raise @@ -91,21 +95,23 @@ class FuseMount(CephFSMount): run_cmd.extend(fuse_cmd) def list_connections(): + from teuthology.misc import get_system_type + + conn_dir = "/sys/fs/fuse/connections" + + self.client_remote.run(args=['sudo', 'modprobe', 'fuse'], + check_status=False) self.client_remote.run( - args=["sudo", "mount", "-t", "fusectl", "/sys/fs/fuse/connections", "/sys/fs/fuse/connections"], - check_status=False, - timeout=(15*60) - ) - p = self.client_remote.run( - args=["ls", "/sys/fs/fuse/connections"], - stdout=StringIO(), - check_status=False, - timeout=(15*60) - ) - if p.exitstatus != 0: + args=["sudo", "mount", "-t", "fusectl", conn_dir, conn_dir], + check_status=False, timeout=(30)) + + try: + ls_str = self.client_remote.sh("ls " + conn_dir, + stdout=StringIO(), + timeout=(15*60)).strip() + except CommandFailedError: return [] - ls_str = p.stdout.getvalue().strip() if ls_str: return [int(n) for n in ls_str.split("\n")] else: @@ -188,25 +194,26 @@ class FuseMount(CephFSMount): self.mountpoint, ], cwd=self.test_dir, - stdout=StringIO(), - stderr=StringIO(), + stdout=BytesIO(), + stderr=BytesIO(), wait=False, timeout=(15*60) ) try: proc.wait() except CommandFailedError: - if ("endpoint is not connected" in proc.stderr.getvalue() - or "Software caused connection abort" in proc.stderr.getvalue()): + error = six.ensure_str(proc.stderr.getvalue()) + if ("endpoint is not connected" in error + or "Software caused connection abort" in error): # This happens is fuse is killed without unmount - log.warn("Found stale moutn point at {0}".format(self.mountpoint)) + log.warning("Found stale moutn point at {0}".format(self.mountpoint)) return True else: # This happens if the mount directory doesn't exist log.info('mount point does not exist: %s', self.mountpoint) return False - fstype = proc.stdout.getvalue().rstrip('\n') + fstype = six.ensure_str(proc.stdout.getvalue()).rstrip('\n') if fstype == 'fuseblk': log.info('ceph-fuse is mounted on %s', self.mountpoint) return True @@ -231,11 +238,11 @@ class FuseMount(CephFSMount): # Now that we're mounted, set permissions so that the rest of the test will have # unrestricted access to the filesystem mount. try: - stderr = StringIO() + stderr = BytesIO() self.client_remote.run(args=['sudo', 'chmod', '1777', self.mountpoint], timeout=(15*60), cwd=self.test_dir, stderr=stderr) except run.CommandFailedError: stderr = stderr.getvalue() - if "Read-only file system".lower() in stderr.lower(): + if b"Read-only file system".lower() in stderr.lower(): pass else: raise @@ -281,7 +288,7 @@ class FuseMount(CephFSMount): """).format(self._fuse_conn)) self._fuse_conn = None - stderr = StringIO() + stderr = BytesIO() try: # make sure its unmounted self.client_remote.run( @@ -348,7 +355,7 @@ class FuseMount(CephFSMount): Prerequisite: the client is not mounted. """ - stderr = StringIO() + stderr = BytesIO() try: self.client_remote.run( args=[ @@ -362,7 +369,7 @@ class FuseMount(CephFSMount): check_status=False, ) except CommandFailedError: - if "No such file or directory" in stderr.getvalue(): + if b"No such file or directory" in stderr.getvalue(): pass else: raise @@ -447,17 +454,18 @@ print(find_socket("{client_name}")) client_name="client.{0}".format(self.client_id)) # Find the admin socket - p = self.client_remote.run(args=[ - 'sudo', 'python3', '-c', pyscript - ], stdout=StringIO(), timeout=(15*60)) - asok_path = p.stdout.getvalue().strip() + asok_path = self.client_remote.sh( + ['sudo', 'python3', '-c', pyscript], + stdout=StringIO(), + timeout=(15*60)).strip() log.info("Found client admin socket at {0}".format(asok_path)) # Query client ID from admin socket - p = self.client_remote.run( - args=['sudo', self._prefix + 'ceph', '--admin-daemon', asok_path] + args, - stdout=StringIO(), timeout=(15*60)) - return json.loads(p.stdout.getvalue()) + json_data = self.client_remote.sh( + ['sudo', self._prefix + 'ceph', '--admin-daemon', asok_path] + args, + stdout=StringIO(), + timeout=(15*60)) + return json.loads(json_data) def get_global_id(self): """ diff --git a/ceph/qa/tasks/cephfs/kernel_mount.py b/ceph/qa/tasks/cephfs/kernel_mount.py index 8e4eeb66a..7ef3f68f8 100644 --- a/ceph/qa/tasks/cephfs/kernel_mount.py +++ b/ceph/qa/tasks/cephfs/kernel_mount.py @@ -1,4 +1,3 @@ -from StringIO import StringIO import json import logging import time @@ -9,7 +8,7 @@ from teuthology import misc from teuthology.orchestra import remote as orchestra_remote from teuthology.orchestra import run from teuthology.contextutil import MaxWhileTries -from .mount import CephFSMount +from tasks.cephfs.mount import CephFSMount log = logging.getLogger(__name__) @@ -214,10 +213,10 @@ class KernelMount(CephFSMount): print(json.dumps(get_id_to_dir())) """) - p = self.client_remote.run(args=[ + output = self.client_remote.sh([ 'sudo', 'python3', '-c', pyscript - ], stdout=StringIO(), timeout=(5*60)) - client_id_to_dir = json.loads(p.stdout.getvalue()) + ], timeout=(5*60)) + client_id_to_dir = json.loads(output) try: return client_id_to_dir[self.client_id] @@ -236,10 +235,10 @@ class KernelMount(CephFSMount): print(open(os.path.join("{debug_dir}", "{filename}")).read()) """).format(debug_dir=debug_dir, filename=filename) - p = self.client_remote.run(args=[ + output = self.client_remote.sh([ 'sudo', 'python3', '-c', pyscript - ], stdout=StringIO(), timeout=(5*60)) - return p.stdout.getvalue() + ], timeout=(5*60)) + return output def get_global_id(self): """ diff --git a/ceph/qa/tasks/cephfs/mount.py b/ceph/qa/tasks/cephfs/mount.py index aeed4fa20..7d04535c8 100644 --- a/ceph/qa/tasks/cephfs/mount.py +++ b/ceph/qa/tasks/cephfs/mount.py @@ -2,10 +2,11 @@ from contextlib import contextmanager import json import logging import datetime +import six import time +from six import StringIO from textwrap import dedent import os -from StringIO import StringIO from teuthology.orchestra import run from teuthology.orchestra.run import CommandFailedError, ConnectionLostError from tasks.cephfs.filesystem import Filesystem @@ -61,6 +62,11 @@ class CephFSMount(object): def mount(self, mount_path=None, mount_fs_name=None, mountpoint=None, mount_options=[]): raise NotImplementedError() + def mount_wait(self, mount_path=None, mount_fs_name=None, mountpoint=None, mount_options=[]): + self.mount(mount_path=mount_path, mount_fs_name=mount_fs_name, mountpoint=mountpoint, + mount_options=mount_options) + self.wait_until_mounted() + def umount(self): raise NotImplementedError() @@ -189,7 +195,7 @@ class CephFSMount(object): def run_python(self, pyscript, py_version='python3'): p = self._run_python(pyscript, py_version) p.wait() - return p.stdout.getvalue().strip() + return six.ensure_str(p.stdout.getvalue().strip()) def run_shell(self, args, wait=True, stdin=None, check_status=True, omit_sudo=True): @@ -418,8 +424,8 @@ class CephFSMount(object): return self.run_shell(["dd", "if=/dev/urandom", "of={0}".format(filename), "bs=1M", "conv=fdatasync", - "count={0}".format(n_mb), - "seek={0}".format(seek) + "count={0}".format(int(n_mb)), + "seek={0}".format(int(seek)) ], wait=wait) def write_test_pattern(self, filename, size): @@ -714,7 +720,7 @@ class CephFSMount(object): else: raise - return p.stdout.getvalue() + return str(p.stdout.getvalue()) def df(self): """ @@ -724,7 +730,7 @@ class CephFSMount(object): p = self.run_shell(["df", "-B1", "."]) lines = p.stdout.getvalue().strip().split("\n") fs, total, used, avail = lines[1].split()[:4] - log.warn(lines) + log.warning(lines) return { "total": int(total), diff --git a/ceph/qa/tasks/cephfs/test_acls.py b/ceph/qa/tasks/cephfs/test_acls.py index 586cbde76..4f704c076 100644 --- a/ceph/qa/tasks/cephfs/test_acls.py +++ b/ceph/qa/tasks/cephfs/test_acls.py @@ -1,7 +1,7 @@ import logging -from StringIO import StringIO -from xfstests_dev import XFSTestsDev +from io import BytesIO +from tasks.cephfs.xfstests_dev import XFSTestsDev log = logging.getLogger(__name__) @@ -22,6 +22,6 @@ class TestACLs(XFSTestsDev): log.info('client is kernel mounted') self.mount_a.client_remote.run(args=['sudo', './check', - 'generic/099'], cwd=self.repo_path, stdout=StringIO(), - stderr=StringIO(), timeout=30, check_status=True, + 'generic/099'], cwd=self.repo_path, stdout=BytesIO(), + stderr=BytesIO(), timeout=30, check_status=True, label='running tests for ACLs from xfstests-dev') diff --git a/ceph/qa/tasks/cephfs/test_admin.py b/ceph/qa/tasks/cephfs/test_admin.py index c53548960..7ff6ed536 100644 --- a/ceph/qa/tasks/cephfs/test_admin.py +++ b/ceph/qa/tasks/cephfs/test_admin.py @@ -7,6 +7,7 @@ from tasks.cephfs.fuse_mount import FuseMount from tasks.cephfs.filesystem import FileLayout + class TestAdminCommands(CephFSTestCase): """ Tests for administration command. @@ -23,6 +24,12 @@ class TestAdminCommands(CephFSTestCase): s = self.fs.mon_manager.raw_cluster_cmd("fs", "status") self.assertTrue("active" in s) + mdsmap = json.loads(self.fs.mon_manager.raw_cluster_cmd("fs", "status", "--format=json-pretty"))["mdsmap"] + self.assertEqual(mdsmap[0]["state"], "active") + + mdsmap = json.loads(self.fs.mon_manager.raw_cluster_cmd("fs", "status", "--format=json"))["mdsmap"] + self.assertEqual(mdsmap[0]["state"], "active") + def _setup_ec_pools(self, n, metadata=True, overwrites=True): if metadata: self.fs.mon_manager.raw_cluster_cmd('osd', 'pool', 'create', n+"-meta", "8") @@ -67,6 +74,14 @@ class TestAdminCommands(CephFSTestCase): self.mount_a.run_shell("mkdir subdir") self.fs.set_dir_layout(self.mount_a, "subdir", FileLayout(pool=p)) + def test_add_data_pool_non_alphamueric_name_as_subdir(self): + """ + That a new data pool with non-alphanumeric name can be added and used for a sub-directory. + """ + p = self.fs.add_data_pool("I-am-data_pool00.") + self.mount_a.run_shell("mkdir subdir") + self.fs.set_dir_layout(self.mount_a, "subdir", FileLayout(pool=p)) + def test_add_data_pool_ec(self): """ That a new EC data pool can be added. diff --git a/ceph/qa/tasks/cephfs/test_auto_repair.py b/ceph/qa/tasks/cephfs/test_auto_repair.py index c0aa2e4c7..141be9883 100644 --- a/ceph/qa/tasks/cephfs/test_auto_repair.py +++ b/ceph/qa/tasks/cephfs/test_auto_repair.py @@ -44,8 +44,7 @@ class TestMDSAutoRepair(CephFSTestCase): self.fs.rados(["rmxattr", dir_objname, "parent"]) # readdir (fetch dirfrag) should fix testdir1's backtrace - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() self.mount_a.run_shell(["ls", "testdir1"]) # flush journal entries to dirfrag objects diff --git a/ceph/qa/tasks/cephfs/test_cephfs_shell.py b/ceph/qa/tasks/cephfs/test_cephfs_shell.py index 6a26cdd9f..1c32de22c 100644 --- a/ceph/qa/tasks/cephfs/test_cephfs_shell.py +++ b/ceph/qa/tasks/cephfs/test_cephfs_shell.py @@ -2,15 +2,16 @@ Before running this testsuite, add path to cephfs-shell module to $PATH and export $PATH. """ +from io import BytesIO from os import path import crypt import logging from tempfile import mkstemp as tempfile_mkstemp import math +from six import ensure_str from sys import version_info as sys_version_info from re import search as re_search from time import sleep -from StringIO import StringIO from tasks.cephfs.cephfs_test_case import CephFSTestCase from teuthology.misc import sudo_write_file from teuthology.orchestra.run import CommandFailedError @@ -52,23 +53,23 @@ class TestCephFSShell(CephFSTestCase): args.extend(("--", cmd)) log.info("Running command: {}".format(" ".join(args))) - return mount_x.client_remote.run(args=args, stdout=StringIO(), - stderr=StringIO(), stdin=stdin) + return mount_x.client_remote.run(args=args, stdout=BytesIO(), + stderr=BytesIO(), stdin=stdin) def get_cephfs_shell_cmd_error(self, cmd, mount_x=None, opts=None, stdin=None): - return self.run_cephfs_shell_cmd(cmd, mount_x, opts, stdin).stderr.\ - getvalue().strip() + return ensure_str(self.run_cephfs_shell_cmd(cmd, mount_x, opts, stdin).stderr.\ + getvalue().strip()) def get_cephfs_shell_cmd_output(self, cmd, mount_x=None, opts=None, stdin=None, config_path=None): - return self.run_cephfs_shell_cmd(cmd, mount_x, opts, stdin, + return ensure_str(self.run_cephfs_shell_cmd(cmd, mount_x, opts, stdin, config_path).\ - stdout.getvalue().strip() + stdout.getvalue().strip()) def get_cephfs_shell_script_output(self, script, mount_x=None, stdin=None): - return self.run_cephfs_shell_script(script, mount_x, stdin).stdout.\ - getvalue().strip() + return ensure_str(self.run_cephfs_shell_script(script, mount_x, stdin).stdout.\ + getvalue().strip()) def run_cephfs_shell_script(self, script, mount_x=None, stdin=None): if mount_x is None: @@ -83,8 +84,8 @@ class TestCephFSShell(CephFSTestCase): args = ["cephfs-shell", "-c", mount_x.config_path, '-b', scriptpath] log.info('Running script \"' + scriptpath + '\"') - return mount_x.client_remote.run(args=args, stdout=StringIO(), - stderr=StringIO(), stdin=stdin) + return mount_x.client_remote.run(args=args, stdout=BytesIO(), + stderr=BytesIO(), stdin=stdin) class TestMkdir(TestCephFSShell): def test_mkdir(self): @@ -490,7 +491,7 @@ class TestDU(TestCephFSShell): du_output = self.get_cephfs_shell_cmd_output('du ' + regfilename) if sys_version_info.major >= 3: - self.assertRegex(expected_output, du_output) + self.assertRegex(du_output, expected_output) elif sys_version_info.major < 3: assert re_search(expected_output, du_output) != None, "\n" + \ "expected_output -\n{}\ndu_output -\n{}\n".format( @@ -512,7 +513,7 @@ class TestDU(TestCephFSShell): sleep(10) du_output = self.get_cephfs_shell_cmd_output('du ' + dirname) if sys_version_info.major >= 3: - self.assertRegex(expected_output, du_output) + self.assertRegex(du_output, expected_output) elif sys_version_info.major < 3: assert re_search(expected_output, du_output) != None, "\n" + \ "expected_output -\n{}\ndu_output -\n{}\n".format( @@ -528,7 +529,7 @@ class TestDU(TestCephFSShell): du_output = self.get_cephfs_shell_cmd_output('du ' + dirname) if sys_version_info.major >= 3: - self.assertRegex(expected_output, du_output) + self.assertRegex(du_output, expected_output) elif sys_version_info.major < 3: assert re_search(expected_output, du_output) != None, "\n" + \ "expected_output -\n{}\ndu_output -\n{}\n".format( @@ -549,7 +550,7 @@ class TestDU(TestCephFSShell): du_output = self.get_cephfs_shell_cmd_output('du ' + hlinkname) if sys_version_info.major >= 3: - self.assertRegex(expected_output, du_output) + self.assertRegex(du_output, expected_output) elif sys_version_info.major < 3: assert re_search(expected_output, du_output) != None, "\n" + \ "expected_output -\n{}\ndu_output -\n{}\n".format( @@ -568,7 +569,7 @@ class TestDU(TestCephFSShell): du_output = self.get_cephfs_shell_cmd_output('du ' + slinkname) if sys_version_info.major >= 3: - self.assertRegex(expected_output, du_output) + self.assertRegex(du_output, expected_output) elif sys_version_info.major < 3: assert re_search(expected_output, du_output) != None, "\n" + \ "expected_output -\n{}\ndu_output -\n{}\n".format( @@ -587,7 +588,7 @@ class TestDU(TestCephFSShell): du_output = self.get_cephfs_shell_cmd_output('du ' + slinkname) if sys_version_info.major >= 3: - self.assertRegex(expected_output, du_output) + self.assertRegex(du_output, expected_output) elif sys_version_info.major < 3: assert re_search(expected_output, du_output) != None, "\n" + \ "expected_output -\n{}\ndu_output -\n{}\n".format( @@ -678,7 +679,7 @@ class TestDU(TestCephFSShell): for expected_output in expected_patterns_in_output: if sys_version_info.major >= 3: - self.assertRegex(expected_output, du_output) + self.assertRegex(du_output, expected_output) elif sys_version_info.major < 3: assert re_search(expected_output, du_output) != None, "\n" + \ "expected_output -\n{}\ndu_output -\n{}\n".format( @@ -695,7 +696,7 @@ class TestDU(TestCephFSShell): for expected_output in expected_patterns_in_output: if sys_version_info.major >= 3: - self.assertRegex(expected_output, du_output) + self.assertRegex(du_output, expected_output) elif sys_version_info.major < 3: assert re_search(expected_output, du_output) != None, "\n" +\ "expected_output -\n{}\ndu_output -\n{}\n".format( @@ -711,7 +712,7 @@ class TestDU(TestCephFSShell): # CWD in DU report. if expected_output.find('/') == len(expected_output) - 1: if sys_version_info.major >= 3: - self.assertRegex(expected_output, du_output) + self.assertRegex(du_output, expected_output) elif sys_version_info.major < 3: assert re_search(expected_output, du_output) != None, "\n" + \ "expected_output -\n{}\ndu_output -\n{}\n".format( @@ -749,7 +750,7 @@ class TestDF(TestCephFSShell): def test_df_for_invalid_directory(self): dir_abspath = path.join(self.mount_a.mountpoint, 'non-existent-dir') proc = self.run_cephfs_shell_cmd('df ' + dir_abspath) - assert proc.stderr.getvalue().find('error in stat') != -1 + assert proc.stderr.getvalue().find(b'error in stat') != -1 def test_df_for_valid_file(self): s = 'df test' * 14145016 @@ -924,12 +925,12 @@ class TestMisc(TestCephFSShell): dirname = 'somedirectory' self.run_cephfs_shell_cmd(['mkdir', dirname]) - output = self.mount_a.client_remote.run(args=['cephfs-shell', '-c', - self.mount_a.config_path, 'ls'], - stdout=StringIO()).stdout.getvalue().strip() + output = self.mount_a.client_remote.sh([ + 'cephfs-shell', '-c', self.mount_a.config_path, 'ls' + ]).strip() if sys_version_info.major >= 3: - self.assertRegex(dirname, output) + self.assertRegex(output, dirname) elif sys_version_info.major < 3: assert re_search(dirname, output) != None, "\n" + \ "expected_output -\n{}\ndu_output -\n{}\n".format( diff --git a/ceph/qa/tasks/cephfs/test_client_limits.py b/ceph/qa/tasks/cephfs/test_client_limits.py index 7b496d751..8491c3d7d 100644 --- a/ceph/qa/tasks/cephfs/test_client_limits.py +++ b/ceph/qa/tasks/cephfs/test_client_limits.py @@ -50,7 +50,7 @@ class TestClientLimits(CephFSTestCase): mds_min_caps_per_client = int(self.fs.get_config("mds_min_caps_per_client")) mds_max_caps_per_client = int(self.fs.get_config("mds_max_caps_per_client")) - mds_recall_warning_decay_rate = self.fs.get_config("mds_recall_warning_decay_rate") + mds_recall_warning_decay_rate = float(self.fs.get_config("mds_recall_warning_decay_rate")) self.assertTrue(open_files >= mds_min_caps_per_client) mount_a_client_id = self.mount_a.get_global_id() @@ -122,8 +122,7 @@ class TestClientLimits(CephFSTestCase): self.set_conf('client.{0}'.format(self.mount_a.client_id), 'client inject release failure', 'true') self.mount_a.teardown() - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() mount_a_client_id = self.mount_a.get_global_id() # Client A creates a file. He will hold the write caps on the file, and later (simulated bug) fail @@ -164,8 +163,7 @@ class TestClientLimits(CephFSTestCase): self.set_conf('client', 'client inject fixed oldest tid', 'true') self.mount_a.teardown() - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() self.fs.mds_asok(['config', 'set', 'mds_max_completed_requests', '{0}'.format(max_requests)]) @@ -176,7 +174,7 @@ class TestClientLimits(CephFSTestCase): self.mount_a.create_n_files("testdir/file2", 5, True) # Wait for the health warnings. Assume mds can handle 10 request per second at least - self.wait_for_health("MDS_CLIENT_OLDEST_TID", max_requests / 10) + self.wait_for_health("MDS_CLIENT_OLDEST_TID", max_requests // 10) def _test_client_cache_size(self, mount_subdir): """ @@ -195,8 +193,7 @@ class TestClientLimits(CephFSTestCase): self.mount_a.run_shell(["mkdir", "subdir"]) self.mount_a.umount_wait() self.set_conf('client', 'client mountpoint', '/subdir') - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() root_ino = self.mount_a.path_to_ino(".") self.assertEqual(root_ino, 1); @@ -217,7 +214,7 @@ class TestClientLimits(CephFSTestCase): self.assertGreaterEqual(dentry_count, num_dirs) self.assertGreaterEqual(dentry_pinned_count, num_dirs) - cache_size = num_dirs / 10 + cache_size = num_dirs // 10 self.mount_a.set_cache_size(cache_size) def trimmed(): diff --git a/ceph/qa/tasks/cephfs/test_client_recovery.py b/ceph/qa/tasks/cephfs/test_client_recovery.py index e18fe997e..9f8aa5dec 100644 --- a/ceph/qa/tasks/cephfs/test_client_recovery.py +++ b/ceph/qa/tasks/cephfs/test_client_recovery.py @@ -103,8 +103,7 @@ class TestClientRecovery(CephFSTestCase): self.mount_b.check_files() - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() # Check that the admin socket interface is correctly reporting # two sessions @@ -160,7 +159,7 @@ class TestClientRecovery(CephFSTestCase): in_reconnect_for = self.fs.wait_for_state('up:active', timeout=self.mds_reconnect_timeout * 2) # Check that the period we waited to enter active is within a factor # of two of the reconnect timeout. - self.assertGreater(in_reconnect_for, self.mds_reconnect_timeout / 2, + self.assertGreater(in_reconnect_for, self.mds_reconnect_timeout // 2, "Should have been in reconnect phase for {0} but only took {1}".format( self.mds_reconnect_timeout, in_reconnect_for )) @@ -169,8 +168,7 @@ class TestClientRecovery(CephFSTestCase): # Check that the client that timed out during reconnect can # mount again and do I/O - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() self.mount_a.create_destroy() self.assert_session_count(2) @@ -212,8 +210,7 @@ class TestClientRecovery(CephFSTestCase): self.mount_a.kill_cleanup() # Bring the client back - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() self.mount_a.create_destroy() def _test_stale_caps(self, write): @@ -226,8 +223,7 @@ class TestClientRecovery(CephFSTestCase): else: self.mount_a.run_shell(["touch", "background_file"]) self.mount_a.umount_wait() - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() cap_holder = self.mount_a.open_background(write=False) self.assert_session_count(2) @@ -444,8 +440,7 @@ class TestClientRecovery(CephFSTestCase): self.mount_a.kill_cleanup() # Bring the client back - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() def test_dir_fsync(self): self._test_fsync(True); @@ -505,8 +500,7 @@ class TestClientRecovery(CephFSTestCase): log.info("Reached active...") # Is the child dentry visible from mount B? - self.mount_b.mount() - self.mount_b.wait_until_mounted() + self.mount_b.mount_wait() self.mount_b.run_shell(["ls", "subdir/childfile"]) def test_unmount_for_evicted_client(self): diff --git a/ceph/qa/tasks/cephfs/test_damage.py b/ceph/qa/tasks/cephfs/test_damage.py index d03e027e7..9a7939219 100644 --- a/ceph/qa/tasks/cephfs/test_damage.py +++ b/ceph/qa/tasks/cephfs/test_damage.py @@ -305,8 +305,7 @@ class TestDamage(CephFSTestCase): log.info("Daemons came up after mutation '{0}', proceeding to ls".format(mutation.desc)) # MDS is up, should go damaged on ls or client mount - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() if mutation.ls_path == ".": proc = self.mount_a.run_shell(["ls", "-R", mutation.ls_path], wait=False) else: @@ -401,8 +400,7 @@ class TestDamage(CephFSTestCase): self.fs.mds_restart() self.fs.wait_for_daemons() - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() dentries = self.mount_a.ls("subdir/") # The damaged guy should have disappeared @@ -461,8 +459,7 @@ class TestDamage(CephFSTestCase): self.assertEqual(scrub_json["raw_stats"]["passed"], False) # Check that the file count is now correct - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() nfiles = self.mount_a.getfattr("./subdir", "ceph.dir.files") self.assertEqual(nfiles, "1") @@ -505,7 +502,7 @@ class TestDamage(CephFSTestCase): self.mds_cluster.mds_fail_restart() self.fs.wait_for_daemons() - self.mount_a.mount() + self.mount_a.mount_wait() # Case 1: un-decodeable backtrace diff --git a/ceph/qa/tasks/cephfs/test_data_scan.py b/ceph/qa/tasks/cephfs/test_data_scan.py index 512e47541..7b6459de9 100644 --- a/ceph/qa/tasks/cephfs/test_data_scan.py +++ b/ceph/qa/tasks/cephfs/test_data_scan.py @@ -7,9 +7,11 @@ import json import logging import os import time -from textwrap import dedent import traceback + +from io import BytesIO from collections import namedtuple, defaultdict +from textwrap import dedent from teuthology.orchestra.run import CommandFailedError from tasks.cephfs.cephfs_test_case import CephFSTestCase, for_teuthology @@ -146,13 +148,13 @@ class StripedStashedLayout(Workload): # Exactly stripe_count objects will exist self.os * self.sc, # Fewer than stripe_count objects will exist - self.os * self.sc / 2, - self.os * (self.sc - 1) + self.os / 2, - self.os * (self.sc - 1) + self.os / 2 - 1, - self.os * (self.sc + 1) + self.os / 2, - self.os * (self.sc + 1) + self.os / 2 + 1, + self.os * self.sc // 2, + self.os * (self.sc - 1) + self.os // 2, + self.os * (self.sc - 1) + self.os // 2 - 1, + self.os * (self.sc + 1) + self.os // 2, + self.os * (self.sc + 1) + self.os // 2 + 1, # More than stripe_count objects will exist - self.os * self.sc + self.os * self.sc / 2 + self.os * self.sc + self.os * self.sc // 2 ] def write(self): @@ -379,8 +381,7 @@ class TestDataScan(CephFSTestCase): log.info(str(self.mds_cluster.status())) # Mount a client - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() # See that the files are present and correct errors = workload.validate() @@ -473,8 +474,7 @@ class TestDataScan(CephFSTestCase): # Start filesystem back up, observe that the file appears to be gone in an `ls` self.fs.mds_restart() self.fs.wait_for_daemons() - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() files = self.mount_a.run_shell(["ls", "subdir/"]).stdout.getvalue().strip().split("\n") self.assertListEqual(sorted(files), sorted(list(set(file_names) - set([victim_dentry])))) @@ -494,8 +494,7 @@ class TestDataScan(CephFSTestCase): # and points to the correct file data. self.fs.mds_restart() self.fs.wait_for_daemons() - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() out = self.mount_a.run_shell(["cat", "subdir/{0}".format(victim_dentry)]).stdout.getvalue().strip() self.assertEqual(out, victim_dentry) @@ -578,7 +577,8 @@ class TestDataScan(CephFSTestCase): # introduce duplicated primary link file1_key = "file1_head" self.assertIn(file1_key, dirfrag1_keys) - file1_omap_data = self.fs.rados(["getomapval", dirfrag1_oid, file1_key, '-']) + file1_omap_data = self.fs.rados(["getomapval", dirfrag1_oid, file1_key, '-'], + stdout_data=BytesIO()) self.fs.rados(["setomapval", dirfrag2_oid, file1_key], stdin_data=file1_omap_data) self.assertIn(file1_key, self._dirfrag_keys(dirfrag2_oid)) @@ -604,8 +604,7 @@ class TestDataScan(CephFSTestCase): self.fs.mds_restart() self.fs.wait_for_daemons() - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() # link count was adjusted? file1_nlink = self.mount_a.path_to_nlink("testdir1/file1") diff --git a/ceph/qa/tasks/cephfs/test_dump_tree.py b/ceph/qa/tasks/cephfs/test_dump_tree.py index 6d943f9dd..48a2c6f00 100644 --- a/ceph/qa/tasks/cephfs/test_dump_tree.py +++ b/ceph/qa/tasks/cephfs/test_dump_tree.py @@ -39,7 +39,7 @@ class TestDumpTree(CephFSTestCase): self.populate() inos = self.get_paths_to_ino() - target = random.choice(inos.keys()) + target = random.sample(inos.keys(), 1)[0] if target != "./": target = os.path.dirname(target) diff --git a/ceph/qa/tasks/cephfs/test_exports.py b/ceph/qa/tasks/cephfs/test_exports.py index 060131add..abaf92e6b 100644 --- a/ceph/qa/tasks/cephfs/test_exports.py +++ b/ceph/qa/tasks/cephfs/test_exports.py @@ -1,6 +1,5 @@ import logging import time -from StringIO import StringIO from tasks.cephfs.fuse_mount import FuseMount from tasks.cephfs.cephfs_test_case import CephFSTestCase @@ -129,15 +128,15 @@ class TestExports(CephFSTestCase): self._wait_subtrees(status, 0, [('/1', 1), ('/1/2', 0), ('/1/2/3', 2)]) if not isinstance(self.mount_a, FuseMount): - p = self.mount_a.client_remote.run(args=['uname', '-r'], stdout=StringIO(), wait=True) + p = self.mount_a.client_remote.sh('uname -r', wait=True) dir_pin = self.mount_a.getfattr("1", "ceph.dir.pin") log.debug("mount.getfattr('1','ceph.dir.pin'): %s " % dir_pin) - if str(p.stdout.getvalue()) < "5" and not(dir_pin): + if str(p) < "5" and not(dir_pin): self.skipTest("Kernel does not support getting the extended attribute ceph.dir.pin") - self.assertTrue(self.mount_a.getfattr("1", "ceph.dir.pin") == "1") - self.assertTrue(self.mount_a.getfattr("1/2", "ceph.dir.pin") == "0") + self.assertEqual(self.mount_a.getfattr("1", "ceph.dir.pin"), '1') + self.assertEqual(self.mount_a.getfattr("1/2", "ceph.dir.pin"), '0') if (len(self.fs.get_active_names()) > 2): - self.assertTrue(self.mount_a.getfattr("1/2/3", "ceph.dir.pin") == "2") + self.assertEqual(self.mount_a.getfattr("1/2/3", "ceph.dir.pin"), '2') def test_session_race(self): """ diff --git a/ceph/qa/tasks/cephfs/test_failover.py b/ceph/qa/tasks/cephfs/test_failover.py index 6872319cf..ec21e6161 100644 --- a/ceph/qa/tasks/cephfs/test_failover.py +++ b/ceph/qa/tasks/cephfs/test_failover.py @@ -3,8 +3,9 @@ import signal import logging import operator from random import randint +from six.moves import range -from cephfs_test_case import CephFSTestCase +from tasks.cephfs.cephfs_test_case import CephFSTestCase from teuthology.exceptions import CommandFailedError from tasks.cephfs.fuse_mount import FuseMount @@ -24,7 +25,7 @@ class TestClusterAffinity(CephFSTestCase): current = sorted(current, key=operator.itemgetter('name')) log.info("current = %s", current) self.assertEqual(len(current), len(target)) - for i in xrange(len(current)): + for i in range(len(current)): for attr in target[i]: self.assertIn(attr, current[i]) self.assertEqual(target[i][attr], current[i][attr]) @@ -413,8 +414,7 @@ class TestFailover(CephFSTestCase): self.mounts[0].umount_wait() # Control: that we can mount and unmount usually, while the cluster is healthy - self.mounts[0].mount() - self.mounts[0].wait_until_mounted() + self.mounts[0].mount_wait() self.mounts[0].umount_wait() # Stop the daemon processes @@ -431,7 +431,7 @@ class TestFailover(CephFSTestCase): self.wait_until_true(laggy, grace * 2) with self.assertRaises(CommandFailedError): - self.mounts[0].mount() + self.mounts[0].mount_wait() def test_standby_count_wanted(self): """ diff --git a/ceph/qa/tasks/cephfs/test_flush.py b/ceph/qa/tasks/cephfs/test_flush.py index ee0b1c92b..17cb84970 100644 --- a/ceph/qa/tasks/cephfs/test_flush.py +++ b/ceph/qa/tasks/cephfs/test_flush.py @@ -88,8 +88,7 @@ class TestFlush(CephFSTestCase): initial_purges = self.fs.mds_asok(['perf', 'dump', 'mds_cache'])['mds_cache']['strays_enqueued'] # Use a client to delete a file - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() self.mount_a.run_shell(["rm", "-rf", "mydir"]) # Flush the journal so that the directory inode can be purged diff --git a/ceph/qa/tasks/cephfs/test_forward_scrub.py b/ceph/qa/tasks/cephfs/test_forward_scrub.py index 6eb31b244..7ed8564d2 100644 --- a/ceph/qa/tasks/cephfs/test_forward_scrub.py +++ b/ceph/qa/tasks/cephfs/test_forward_scrub.py @@ -10,7 +10,10 @@ how the functionality responds to damaged metadata. import json import logging +import six + from collections import namedtuple +from io import BytesIO from textwrap import dedent from teuthology.orchestra.run import CommandFailedError @@ -31,9 +34,10 @@ class TestForwardScrub(CephFSTestCase): """ Read a ceph-encoded string from a rados xattr """ - output = self.fs.rados(["getxattr", obj, attr], pool=pool) + output = self.fs.rados(["getxattr", obj, attr], pool=pool, + stdout_data=BytesIO()) strlen = struct.unpack('i', output[0:4])[0] - return output[4:(4 + strlen)] + return six.ensure_str(output[4:(4 + strlen)], encoding='ascii') def _get_paths_to_ino(self): inos = {} @@ -132,7 +136,7 @@ class TestForwardScrub(CephFSTestCase): # Create a new inode that's just in the log, i.e. would # look orphaned to backward scan if backward scan wisnae # respectin' tha scrub_tag xattr. - self.mount_a.mount() + self.mount_a.mount_wait() self.mount_a.run_shell(["mkdir", "parent/unflushed"]) self.mount_a.run_shell(["dd", "if=/dev/urandom", "of=./parent/unflushed/jfile", @@ -155,7 +159,7 @@ class TestForwardScrub(CephFSTestCase): self.fs.wait_for_daemons() # See that the orphaned file is indeed missing from a client's POV - self.mount_a.mount() + self.mount_a.mount_wait() damaged_state = self._get_paths_to_ino() self.assertNotIn("./parent/flushed/bravo", damaged_state) self.mount_a.umount_wait() @@ -192,7 +196,7 @@ class TestForwardScrub(CephFSTestCase): # and no lost+found, and no extra inodes! self.fs.mds_restart() self.fs.wait_for_daemons() - self.mount_a.mount() + self.mount_a.mount_wait() self._validate_linkage(inos) def _stash_inotable(self): @@ -218,7 +222,7 @@ class TestForwardScrub(CephFSTestCase): inotable_copy = self._stash_inotable() - self.mount_a.mount() + self.mount_a.mount_wait() self.mount_a.write_n_mb("file2_sixmegs", 6) self.mount_a.write_n_mb("file3_sixmegs", 6) diff --git a/ceph/qa/tasks/cephfs/test_full.py b/ceph/qa/tasks/cephfs/test_full.py index 3ba05af1d..eaa36c7c9 100644 --- a/ceph/qa/tasks/cephfs/test_full.py +++ b/ceph/qa/tasks/cephfs/test_full.py @@ -125,9 +125,9 @@ class FullnessTestCase(CephFSTestCase): # Fill up the cluster. This dd may or may not fail, as it depends on # how soon the cluster recognises its own fullness - self.mount_a.write_n_mb("large_file_a", self.fill_mb / 2) + self.mount_a.write_n_mb("large_file_a", self.fill_mb // 2) try: - self.mount_a.write_n_mb("large_file_b", self.fill_mb / 2) + self.mount_a.write_n_mb("large_file_b", self.fill_mb // 2) except CommandFailedError: log.info("Writing file B failed (full status happened already)") assert self.is_full() @@ -138,7 +138,7 @@ class FullnessTestCase(CephFSTestCase): # Attempting to write more data should give me ENOSPC with self.assertRaises(CommandFailedError) as ar: - self.mount_a.write_n_mb("large_file_b", 50, seek=self.fill_mb / 2) + self.mount_a.write_n_mb("large_file_b", 50, seek=self.fill_mb // 2) self.assertEqual(ar.exception.exitstatus, 1) # dd returns 1 on "No space" # Wait for the MDS to see the latest OSD map so that it will reliably @@ -212,7 +212,7 @@ class FullnessTestCase(CephFSTestCase): # Configs for this test should bring this setting down in order to # run reasonably quickly if osd_mon_report_interval > 10: - log.warn("This test may run rather slowly unless you decrease" + log.warning("This test may run rather slowly unless you decrease" "osd_mon_report_interval (5 is a good setting)!") self.mount_a.run_python(template.format( @@ -361,7 +361,7 @@ class TestQuotaFull(FullnessTestCase): Test per-pool fullness, which indicates quota limits exceeded """ pool_capacity = 1024 * 1024 * 32 # arbitrary low-ish limit - fill_mb = pool_capacity / (1024 * 1024) # type: ignore + fill_mb = pool_capacity // (1024 * 1024) # type: ignore # We are only testing quota handling on the data pool, not the metadata # pool. @@ -392,7 +392,7 @@ class TestClusterFull(FullnessTestCase): max_avail = self.fs.get_pool_df(self._data_pool_name())['max_avail'] full_ratio = float(self.fs.get_config("mon_osd_full_ratio", service_type="mon")) TestClusterFull.pool_capacity = int(max_avail * full_ratio) - TestClusterFull.fill_mb = (self.pool_capacity / (1024 * 1024)) + TestClusterFull.fill_mb = (self.pool_capacity // (1024 * 1024)) def is_full(self): return self.fs.is_full() diff --git a/ceph/qa/tasks/cephfs/test_journal_migration.py b/ceph/qa/tasks/cephfs/test_journal_migration.py index 9d1d399ca..8863b371f 100644 --- a/ceph/qa/tasks/cephfs/test_journal_migration.py +++ b/ceph/qa/tasks/cephfs/test_journal_migration.py @@ -1,5 +1,4 @@ -from StringIO import StringIO from tasks.cephfs.cephfs_test_case import CephFSTestCase from tasks.workunit import task as workunit @@ -76,14 +75,12 @@ class TestJournalMigration(CephFSTestCase): self.fs.journal_tool(["event", "get", "json", "--path", "/tmp/journal.json"], 0) - p = self.fs.tool_remote.run( - args=[ + p = self.fs.tool_remote.sh([ "python3", "-c", "import json; print(len(json.load(open('/tmp/journal.json'))))" - ], - stdout=StringIO()) - event_count = int(p.stdout.getvalue().strip()) + ]) + event_count = int(p.strip()) if event_count < 1000: # Approximate value of "lots", expected from having run fsstress raise RuntimeError("Unexpectedly few journal events: {0}".format(event_count)) diff --git a/ceph/qa/tasks/cephfs/test_journal_repair.py b/ceph/qa/tasks/cephfs/test_journal_repair.py index a52455d71..61037b96d 100644 --- a/ceph/qa/tasks/cephfs/test_journal_repair.py +++ b/ceph/qa/tasks/cephfs/test_journal_repair.py @@ -92,8 +92,7 @@ class TestJournalRepair(CephFSTestCase): self.fs.wait_for_daemons() # List files - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() # First ls -R to populate MDCache, such that hardlinks will # resolve properly (recover_dentries does not create backtraces, @@ -102,8 +101,7 @@ class TestJournalRepair(CephFSTestCase): # FIXME: hook in forward scrub here to regenerate backtraces proc = self.mount_a.run_shell(['ls', '-R']) self.mount_a.umount_wait() # remount to clear client cache before our second ls - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() proc = self.mount_a.run_shell(['ls', '-R']) self.assertEqual(proc.stdout.getvalue().strip(), @@ -278,7 +276,7 @@ class TestJournalRepair(CephFSTestCase): self.fs.mds_fail_restart(active_mds_names[0]) self.wait_until_equal(lambda: self.fs.get_active_names(), [active_mds_names[0]], 30, reject_fn=lambda v: len(v) > 1) - self.mount_a.mount() + self.mount_a.mount_wait() self.mount_a.run_shell(["ls", "-R"], wait=True) def test_table_tool(self): @@ -434,7 +432,7 @@ class TestJournalRepair(CephFSTestCase): self.fs.mds_restart() self.fs.wait_for_daemons() - self.mount_a.mount() + self.mount_a.mount_wait() # trivial sync moutn a workunit(self.ctx, { diff --git a/ceph/qa/tasks/cephfs/test_misc.py b/ceph/qa/tasks/cephfs/test_misc.py index 342fb76d4..c73284558 100644 --- a/ceph/qa/tasks/cephfs/test_misc.py +++ b/ceph/qa/tasks/cephfs/test_misc.py @@ -25,8 +25,7 @@ class TestMisc(CephFSTestCase): # on lookup/open self.mount_b.umount_wait() self.set_conf('client', 'client debug getattr caps', 'true') - self.mount_b.mount() - self.mount_b.wait_until_mounted() + self.mount_b.mount_wait() # create a file and hold it open. MDS will issue CEPH_CAP_EXCL_* # to mount_a @@ -47,7 +46,7 @@ class TestMisc(CephFSTestCase): t = time.time() rctime = self.mount_a.getfattr(".", "ceph.dir.rctime") log.info("rctime = {}".format(rctime)) - self.assertGreaterEqual(rctime, t-10) + self.assertGreaterEqual(float(rctime), t - 10) def test_fs_new(self): self.mount_a.umount_wait() diff --git a/ceph/qa/tasks/cephfs/test_openfiletable.py b/ceph/qa/tasks/cephfs/test_openfiletable.py index 24f84e842..d64c7d94f 100644 --- a/ceph/qa/tasks/cephfs/test_openfiletable.py +++ b/ceph/qa/tasks/cephfs/test_openfiletable.py @@ -1,6 +1,6 @@ import time import logging -from cephfs_test_case import CephFSTestCase +from tasks.cephfs.cephfs_test_case import CephFSTestCase from teuthology.exceptions import CommandFailedError from tasks.cephfs.cephfs_test_case import CephFSTestCase, for_teuthology @@ -31,7 +31,7 @@ class OpenFileTable(CephFSTestCase): # Hold the file open file_count = 8 - for i in xrange(0, file_count): + for i in range(0, file_count): filename = "open_file{}".format(i) p = self.mount_a.open_background(filename) self.mount_a.write_n_mb(filename, size_mb) diff --git a/ceph/qa/tasks/cephfs/test_pool_perm.py b/ceph/qa/tasks/cephfs/test_pool_perm.py index a1f234a22..9912debed 100644 --- a/ceph/qa/tasks/cephfs/test_pool_perm.py +++ b/ceph/qa/tasks/cephfs/test_pool_perm.py @@ -35,8 +35,7 @@ class TestPoolPerm(CephFSTestCase): 'allow r pool={0}'.format(self.fs.get_data_pool_name())) self.mount_a.umount_wait() - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() # write should fail self.mount_a.run_python(remote_script.format(path=file_path, check_read=str(False))) @@ -47,8 +46,7 @@ class TestPoolPerm(CephFSTestCase): 'allow w pool={0}'.format(self.fs.get_data_pool_name())) self.mount_a.umount_wait() - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() # read should fail self.mount_a.run_python(remote_script.format(path=file_path, check_read=str(True))) @@ -77,8 +75,7 @@ class TestPoolPerm(CephFSTestCase): )) self.mount_a.umount_wait() - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() with self.assertRaises(CommandFailedError): self.mount_a.setfattr("layoutfile", "ceph.file.layout.pool", @@ -96,8 +93,7 @@ class TestPoolPerm(CephFSTestCase): self.fs.get_data_pool_names()[0], self.fs.get_data_pool_names()[1], )) - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() self.mount_a.setfattr("layoutfile", "ceph.file.layout.pool", new_pool_name) self.mount_a.setfattr("layoutdir", "ceph.dir.layout.pool", diff --git a/ceph/qa/tasks/cephfs/test_quota.py b/ceph/qa/tasks/cephfs/test_quota.py index ee11c586e..dcfda5e21 100644 --- a/ceph/qa/tasks/cephfs/test_quota.py +++ b/ceph/qa/tasks/cephfs/test_quota.py @@ -1,5 +1,5 @@ -from cephfs_test_case import CephFSTestCase +from tasks.cephfs.cephfs_test_case import CephFSTestCase from teuthology.exceptions import CommandFailedError diff --git a/ceph/qa/tasks/cephfs/test_readahead.py b/ceph/qa/tasks/cephfs/test_readahead.py index 31e7bf187..e936a94b9 100644 --- a/ceph/qa/tasks/cephfs/test_readahead.py +++ b/ceph/qa/tasks/cephfs/test_readahead.py @@ -15,8 +15,7 @@ class TestReadahead(CephFSTestCase): # Unmount and remount the client to flush cache self.mount_a.umount_wait() - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() initial_op_r = self.mount_a.admin_socket(['perf', 'dump', 'objecter'])['objecter']['op_r'] self.mount_a.run_shell(["dd", "if=foo", "of=/dev/null", "bs=128k", "count=32"]) diff --git a/ceph/qa/tasks/cephfs/test_recovery_pool.py b/ceph/qa/tasks/cephfs/test_recovery_pool.py index 36b4e58ec..0b3dc56bf 100644 --- a/ceph/qa/tasks/cephfs/test_recovery_pool.py +++ b/ceph/qa/tasks/cephfs/test_recovery_pool.py @@ -186,10 +186,8 @@ class TestRecoveryPool(CephFSTestCase): log.info(str(self.mds_cluster.status())) # Mount a client - self.mount_a.mount() - self.mount_b.mount(mount_fs_name=recovery_fs) - self.mount_a.wait_until_mounted() - self.mount_b.wait_until_mounted() + self.mount_a.mount_wait() + self.mount_b.mount_wait(mount_fs_name=recovery_fs) # See that the files are present and correct errors = workload.validate() diff --git a/ceph/qa/tasks/cephfs/test_scrub.py b/ceph/qa/tasks/cephfs/test_scrub.py index e4f0cb9be..1875b5f34 100644 --- a/ceph/qa/tasks/cephfs/test_scrub.py +++ b/ceph/qa/tasks/cephfs/test_scrub.py @@ -13,6 +13,7 @@ ValidationError = namedtuple("ValidationError", ["exception", "backtrace"]) class Workload(CephFSTestCase): def __init__(self, filesystem, mount): + super().__init__() self._mount = mount self._filesystem = filesystem self._initial_state = None @@ -74,6 +75,9 @@ class BacktraceWorkload(Workload): self._filesystem.mds_asok(["flush", "journal"]) self._filesystem._write_data_xattr(st['st_ino'], "parent", "") + def create_files(self, nfiles=1000): + self._mount.create_n_files("scrub-new-files/file", nfiles) + class DupInodeWorkload(Workload): """ @@ -88,7 +92,7 @@ class DupInodeWorkload(Workload): def damage(self): temp_bin_path = "/tmp/10000000000.00000000_omap.bin" - self._mount.umount() + self._mount.umount_wait() self._filesystem.mds_asok(["flush", "journal"]) self._filesystem.mds_stop() self._filesystem.rados(["getomapval", "10000000000.00000000", @@ -109,6 +113,9 @@ class DupInodeWorkload(Workload): class TestScrub(CephFSTestCase): MDSS_REQUIRED = 1 + def setUp(self): + super().setUp() + def _scrub(self, workload, workers=1): """ That when all objects in metadata pool are removed, we can rebuild a metadata pool @@ -140,6 +147,27 @@ class TestScrub(CephFSTestCase): errors[0].exception, errors[0].backtrace )) + def _get_damage_count(self, damage_type='backtrace'): + out_json = self.fs.rank_tell(["damage", "ls"]) + self.assertNotEqual(out_json, None) + + damage_count = 0 + for it in out_json: + if it['damage_type'] == damage_type: + damage_count += 1 + return damage_count + + def _scrub_new_files(self, workload): + """ + That scrubbing new files does not lead to errors + """ + workload.create_files(1000) + self._wait_until_scrub_complete() + self.assertEqual(self._get_damage_count(), 0) + + def test_scrub_backtrace_for_new_files(self): + self._scrub_new_files(BacktraceWorkload(self.fs, self.mount_a)) + def test_scrub_backtrace(self): self._scrub(BacktraceWorkload(self.fs, self.mount_a)) diff --git a/ceph/qa/tasks/cephfs/test_scrub_checks.py b/ceph/qa/tasks/cephfs/test_scrub_checks.py index 012b6c009..b6ca0b898 100644 --- a/ceph/qa/tasks/cephfs/test_scrub_checks.py +++ b/ceph/qa/tasks/cephfs/test_scrub_checks.py @@ -298,8 +298,7 @@ class TestScrubChecks(CephFSTestCase): self.fs.mds_fail_restart() self.fs.wait_for_daemons() - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() # fragstat indicates the directory is not empty, rmdir should fail with self.assertRaises(CommandFailedError) as ar: diff --git a/ceph/qa/tasks/cephfs/test_sessionmap.py b/ceph/qa/tasks/cephfs/test_sessionmap.py index a4642de0b..bdcde71d0 100644 --- a/ceph/qa/tasks/cephfs/test_sessionmap.py +++ b/ceph/qa/tasks/cephfs/test_sessionmap.py @@ -62,8 +62,7 @@ class TestSessionMap(CephFSTestCase): status = self.fs.status() s = self._get_connection_count(status=status) - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() self.assertGreater(self._get_connection_count(status=status), s) self.mount_a.umount_wait() e = self._get_connection_count(status=status) @@ -85,8 +84,8 @@ class TestSessionMap(CephFSTestCase): status = self.fs.wait_for_daemons() # Bring the clients back - self.mount_a.mount() - self.mount_b.mount() + self.mount_a.mount_wait() + self.mount_b.mount_wait() # See that they've got sessions self.assert_session_count(2, mds_id=self.fs.get_rank(status=status)['name']) @@ -178,8 +177,7 @@ class TestSessionMap(CephFSTestCase): # Configure a client that is limited to /foo/bar self._configure_auth(self.mount_b, "badguy", "allow rw path=/foo/bar") # Check he can mount that dir and do IO - self.mount_b.mount(mount_path="/foo/bar") - self.mount_b.wait_until_mounted() + self.mount_b.mount_wait(mount_path="/foo/bar") self.mount_b.create_destroy() self.mount_b.umount_wait() @@ -225,5 +223,4 @@ class TestSessionMap(CephFSTestCase): self.assert_session_count(1, mds_id=self.fs.get_rank(rank=1, status=status)['name']) self.mount_a.kill_cleanup() - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() diff --git a/ceph/qa/tasks/cephfs/test_snapshots.py b/ceph/qa/tasks/cephfs/test_snapshots.py index a1dcc23d3..0a35d99d4 100644 --- a/ceph/qa/tasks/cephfs/test_snapshots.py +++ b/ceph/qa/tasks/cephfs/test_snapshots.py @@ -131,8 +131,7 @@ class TestSnapshots(CephFSTestCase): else: self.assertGreater(self._get_last_created_snap(rank=0), last_created) - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() self.mount_a.run_shell(["rmdir", Raw("d1/dir/.snap/*")]) @@ -173,8 +172,7 @@ class TestSnapshots(CephFSTestCase): else: self.assertGreater(self._get_last_created_snap(rank=0), last_created) - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() self.mount_a.run_shell(["rmdir", Raw("d1/dir/.snap/*")]) @@ -216,8 +214,7 @@ class TestSnapshots(CephFSTestCase): self.wait_until_true(lambda: len(self._get_pending_snap_update(rank=0)) == 0, timeout=30) self.assertEqual(self._get_last_created_snap(rank=0), last_created) - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() def test_snapclient_cache(self): """ @@ -345,8 +342,7 @@ class TestSnapshots(CephFSTestCase): self.assertEqual(snaps_dump["last_created"], rank0_cache["last_created"]) self.assertTrue(_check_snapclient_cache(snaps_dump, cache_dump=rank0_cache)); - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() self.mount_a.run_shell(["rmdir", Raw("d0/d2/dir/.snap/*")]) @@ -474,7 +470,6 @@ class TestSnapshots(CephFSTestCase): # failing at the last mkdir beyond the limit is expected if sno == snaps: log.info("failed while creating snap #{}: {}".format(sno, repr(e))) - sys.exc_clear() raise TestSnapshots.SnapLimitViolationException(sno) def test_mds_max_snaps_per_dir_default_limit(self): @@ -500,7 +495,6 @@ class TestSnapshots(CephFSTestCase): self.create_dir_and_snaps("accounts", new_limit + 1) except TestSnapshots.SnapLimitViolationException as e: if e.failed_snapshot_number == (new_limit + 1): - sys.exc_clear() pass # then increase the limit by one and test new_limit = new_limit + 1 diff --git a/ceph/qa/tasks/cephfs/test_strays.py b/ceph/qa/tasks/cephfs/test_strays.py index e02eca2d6..a5058441e 100644 --- a/ceph/qa/tasks/cephfs/test_strays.py +++ b/ceph/qa/tasks/cephfs/test_strays.py @@ -152,7 +152,7 @@ class TestStrays(CephFSTestCase): os.mkdir(os.path.join(mount_path, subdir)) for i in range(0, file_multiplier): for size in range(0, {size_range}*size_unit, size_unit): - filename = "{{0}}_{{1}}.bin".format(i, size / size_unit) + filename = "{{0}}_{{1}}.bin".format(i, size // size_unit) with open(os.path.join(mount_path, subdir, filename), 'w') as f: f.write(size * 'x') """.format( @@ -237,7 +237,7 @@ class TestStrays(CephFSTestCase): # insanely fast such that the deletions all pass before we have polled the # statistics. if throttle_type == self.OPS_THROTTLE: - if ops_high_water < mds_max_purge_ops / 2: + if ops_high_water < mds_max_purge_ops // 2: raise RuntimeError("Ops in flight high water is unexpectedly low ({0} / {1})".format( ops_high_water, mds_max_purge_ops )) @@ -248,7 +248,7 @@ class TestStrays(CephFSTestCase): # particularly large file/directory. self.assertLessEqual(ops_high_water, mds_max_purge_ops+64) elif throttle_type == self.FILES_THROTTLE: - if files_high_water < mds_max_purge_files / 2: + if files_high_water < mds_max_purge_files // 2: raise RuntimeError("Files in flight high water is unexpectedly low ({0} / {1})".format( files_high_water, mds_max_purge_files )) @@ -373,7 +373,7 @@ class TestStrays(CephFSTestCase): self.fs.mds_asok(['flush', 'journal']) self.fs.mds_fail_restart() self.fs.wait_for_daemons() - self.mount_a.mount() + self.mount_a.mount_wait() # Unlink file_a self.mount_a.run_shell(["rm", "-f", "dir_1/file_a"]) @@ -628,7 +628,7 @@ class TestStrays(CephFSTestCase): rank_0_id = active_mds_names[0] rank_1_id = active_mds_names[1] - self.mount_a.mount() + self.mount_a.mount_wait() self.mount_a.run_shell(["rm", "-f", "dir_1/original"]) self.mount_a.umount_wait() @@ -772,7 +772,7 @@ class TestStrays(CephFSTestCase): # zero, but there's actually still a stray, so at the very # least the StrayManager stats code is slightly off - self.mount_a.mount() + self.mount_a.mount_wait() # See that the data from the snapshotted revision of the file is still present # and correct @@ -873,8 +873,7 @@ class TestStrays(CephFSTestCase): # remount+flush (release client caps) self.mount_a.umount_wait() self.fs.mds_asok(["flush", "journal"], mds_id) - self.mount_a.mount() - self.mount_a.wait_until_mounted() + self.mount_a.mount_wait() # Create 50% more files than the current fragment limit self.mount_a.run_python(dedent(""" diff --git a/ceph/qa/tasks/cephfs/test_volume_client.py b/ceph/qa/tasks/cephfs/test_volume_client.py index a6b4d10af..e02ba0ee7 100644 --- a/ceph/qa/tasks/cephfs/test_volume_client.py +++ b/ceph/qa/tasks/cephfs/test_volume_client.py @@ -390,13 +390,13 @@ vc.disconnect() m.umount_wait() # Create a dir on mount A - self.mount_a.mount() + self.mount_a.mount_wait() self.mount_a.run_shell(["mkdir", "parent1"]) self.mount_a.run_shell(["mkdir", "parent2"]) self.mount_a.run_shell(["mkdir", "parent1/mydir"]) # Put some files in it from mount B - self.mount_b.mount() + self.mount_b.mount_wait() self.mount_b.run_shell(["touch", "parent1/mydir/afile"]) self.mount_b.umount_wait() @@ -606,8 +606,10 @@ vc.disconnect() # Read existing content of the volume. self.assertListEqual(guest_mount.ls(guest_mount.mountpoint), ["data.bin"]) # Cannot write into read-only volume. - with self.assertRaises(CommandFailedError): + try: guest_mount.write_n_mb("rogue.bin", 1) + except CommandFailedError: + pass def test_get_authorized_ids(self): """ @@ -656,10 +658,10 @@ vc.disconnect() # Check the list of authorized IDs and their access levels. if self.py_version == 'python3': expected_result = [('guest1', 'rw'), ('guest2', 'r')] + self.assertCountEqual(str(expected_result), auths) else: expected_result = [(u'guest1', u'rw'), (u'guest2', u'r')] - - self.assertItemsEqual(str(expected_result), auths) + self.assertItemsEqual(str(expected_result), auths) # Disallow both the auth IDs' access to the volume. auths = self._volume_client_python(volumeclient_mount, dedent(""" diff --git a/ceph/qa/tasks/cephfs/test_volumes.py b/ceph/qa/tasks/cephfs/test_volumes.py index 9fca6de56..11c23605a 100644 --- a/ceph/qa/tasks/cephfs/test_volumes.py +++ b/ceph/qa/tasks/cephfs/test_volumes.py @@ -164,6 +164,14 @@ class TestVolumes(CephFSTestCase): subvol_md = self._fs_cmd(*args) return subvol_md + def _get_subvolume_snapshot_info(self, vol_name, subvol_name, snapname, group_name=None): + args = ["subvolume", "snapshot", "info", vol_name, subvol_name, snapname] + if group_name: + args.append(group_name) + args = tuple(args) + snap_md = self._fs_cmd(*args) + return snap_md + def _delete_test_volume(self): self._fs_cmd("volume", "rm", self.volname, "--yes-i-really-mean-it") @@ -364,7 +372,7 @@ class TestVolumes(CephFSTestCase): self.assertNotEqual(subvolpath, None) # shrink the subvolume - nsize = osize/2 + nsize = osize // 2 self._fs_cmd("subvolume", "resize", self.volname, subvolname, str(nsize)) # verify the quota @@ -456,7 +464,7 @@ class TestVolumes(CephFSTestCase): self.assertEqual(usedsize, susedsize) # shrink the subvolume - nsize = usedsize/2 + nsize = usedsize // 2 try: self._fs_cmd("subvolume", "resize", self.volname, subvolname, str(nsize)) except CommandFailedError: @@ -496,7 +504,7 @@ class TestVolumes(CephFSTestCase): self.assertEqual(usedsize, susedsize) # shrink the subvolume - nsize = usedsize/2 + nsize = usedsize // 2 try: self._fs_cmd("subvolume", "resize", self.volname, subvolname, str(nsize), "--no_shrink") except CommandFailedError as ce: @@ -531,7 +539,7 @@ class TestVolumes(CephFSTestCase): self.mount_a.write_n_mb(os.path.join(subvolpath, filename), file_size) # create a file of size 5MB and try write more - file_size=file_size/2 + file_size=file_size // 2 number_of_files=1 log.debug("filling subvolume {0} with {1} file of size {2}MB".format(subvolname, number_of_files, @@ -566,6 +574,44 @@ class TestVolumes(CephFSTestCase): # verify trash dir is clean self._wait_for_trash_empty() + def test_subvolume_create_idempotence_resize(self): + # create subvolume + subvolume = self._generate_random_subvolume_name() + self._fs_cmd("subvolume", "create", self.volname, subvolume) + + # try creating w/ same subvolume name with size -- should set quota + self._fs_cmd("subvolume", "create", self.volname, subvolume, "1000000000") + + # get subvolume metadata + subvol_info = json.loads(self._get_subvolume_info(self.volname, subvolume)) + self.assertEqual(subvol_info["bytes_quota"], 1000000000) + + # remove subvolume + self._fs_cmd("subvolume", "rm", self.volname, subvolume) + + # verify trash dir is clean + self._wait_for_trash_empty() + + def test_subvolume_create_isolated_namespace(self): + """ + Create subvolume in separate rados namespace + """ + + # create subvolume + subvolume = self._generate_random_subvolume_name() + self._fs_cmd("subvolume", "create", self.volname, subvolume, "--namespace-isolated") + + # get subvolume metadata + subvol_info = json.loads(self._get_subvolume_info(self.volname, subvolume)) + self.assertNotEqual(len(subvol_info), 0) + self.assertEqual(subvol_info["pool_namespace"], "fsvolumens_" + subvolume) + + # remove subvolumes + self._fs_cmd("subvolume", "rm", self.volname, subvolume) + + # verify trash dir is clean + self._wait_for_trash_empty() + def test_subvolume_create_with_invalid_data_pool_layout(self): subvolume = self._generate_random_subvolume_name() data_pool = "invalid_pool" @@ -745,7 +791,8 @@ class TestVolumes(CephFSTestCase): # tests the 'fs subvolume info' command subvol_md = ["atime", "bytes_pcent", "bytes_quota", "bytes_used", "created_at", "ctime", - "data_pool", "gid", "mode", "mon_addrs", "mtime", "path", "type", "uid"] + "data_pool", "gid", "mode", "mon_addrs", "mtime", "path", "pool_namespace", + "type", "uid"] # create subvolume subvolume = self._generate_random_subvolume_name() @@ -764,6 +811,7 @@ class TestVolumes(CephFSTestCase): if subvol_info["bytes_quota"] != "infinite": raise RuntimeError("bytes_quota should be set to infinite if quota is not set") + self.assertEqual(subvol_info["pool_namespace"], "") nsize = self.DEFAULT_FILE_SIZE*1024*1024 try: @@ -794,7 +842,8 @@ class TestVolumes(CephFSTestCase): # tests the 'fs subvolume info' command for a clone subvol_md = ["atime", "bytes_pcent", "bytes_quota", "bytes_used", "created_at", "ctime", - "data_pool", "gid", "mode", "mon_addrs", "mtime", "path", "type", "uid"] + "data_pool", "gid", "mode", "mon_addrs", "mtime", "path", "pool_namespace", + "type", "uid"] subvolume = self._generate_random_subvolume_name() snapshot = self._generate_random_snapshot_name() @@ -1143,6 +1192,49 @@ class TestVolumes(CephFSTestCase): # verify trash dir is clean self._wait_for_trash_empty() + def test_subvolume_snapshot_info(self): + + """ + tests the 'fs subvolume snapshot info' command + """ + + snap_metadata = ["created_at", "data_pool", "has_pending_clones", "protected", "size"] + + subvolume = self._generate_random_subvolume_name() + snapshot = self._generate_random_snapshot_name() + + # create subvolume + self._fs_cmd("subvolume", "create", self.volname, subvolume) + + # do some IO + self._do_subvolume_io(subvolume, number_of_files=1) + + # snapshot subvolume + self._fs_cmd("subvolume", "snapshot", "create", self.volname, subvolume, snapshot) + + # now, protect snapshot + self._fs_cmd("subvolume", "snapshot", "protect", self.volname, subvolume, snapshot) + + snap_info = json.loads(self._get_subvolume_snapshot_info(self.volname, subvolume, snapshot)) + self.assertNotEqual(len(snap_info), 0) + for md in snap_metadata: + if md not in snap_info: + raise RuntimeError("%s not present in the metadata of subvolume snapshot" % md) + self.assertEqual(snap_info["protected"], "yes") + self.assertEqual(snap_info["has_pending_clones"], "no") + + # now, unprotect snapshot + self._fs_cmd("subvolume", "snapshot", "unprotect", self.volname, subvolume, snapshot) + + # remove snapshot + self._fs_cmd("subvolume", "snapshot", "rm", self.volname, subvolume, snapshot) + + # remove subvolume + self._fs_cmd("subvolume", "rm", self.volname, subvolume) + + # verify trash dir is clean + self._wait_for_trash_empty() + def test_subvolume_snapshot_create_idempotence(self): subvolume = self._generate_random_subvolume_name() snapshot = self._generate_random_snapshot_name() @@ -1388,7 +1480,7 @@ class TestVolumes(CephFSTestCase): for subvolume in subvolumes: self._fs_cmd("subvolume", "rm", self.volname, subvolume) - self.mount_a.mount() + self.mount_a.mount_wait() # verify trash dir is clean self._wait_for_trash_empty(timeout=300) @@ -2156,7 +2248,7 @@ class TestVolumes(CephFSTestCase): self.fs.add_data_pool(new_pool) self.fs.mon_manager.raw_cluster_cmd("osd", "pool", "set-quota", new_pool, - "max_bytes", "{0}".format(pool_capacity / 4)) + "max_bytes", "{0}".format(pool_capacity // 4)) # schedule a clone self._fs_cmd("subvolume", "snapshot", "clone", self.volname, subvolume, snapshot, clone1, "--pool_layout", new_pool) diff --git a/ceph/qa/tasks/cephfs/xfstests_dev.py b/ceph/qa/tasks/cephfs/xfstests_dev.py index c11561fae..1a18b7e14 100644 --- a/ceph/qa/tasks/cephfs/xfstests_dev.py +++ b/ceph/qa/tasks/cephfs/xfstests_dev.py @@ -1,6 +1,6 @@ +from io import BytesIO import six import logging -from StringIO import StringIO from tasks.cephfs.cephfs_test_case import CephFSTestCase logger = logging.getLogger(__name__) @@ -28,11 +28,11 @@ class XFSTestsDev(CephFSTestCase): # NOTE: On teuthology machines it's necessary to run "make" as # superuser since the repo is cloned somewhere in /tmp. self.mount_a.client_remote.run(args=['sudo', 'make'], - cwd=self.repo_path, stdout=StringIO(), - stderr=StringIO()) + cwd=self.repo_path, stdout=BytesIO(), + stderr=BytesIO()) self.mount_a.client_remote.run(args=['sudo', 'make', 'install'], cwd=self.repo_path, omit_sudo=False, - stdout=StringIO(), stderr=StringIO()) + stdout=BytesIO(), stderr=BytesIO()) def get_repo(self): """ diff --git a/ceph/qa/tasks/check_counter.py b/ceph/qa/tasks/check_counter.py index fc877f285..daa81973b 100644 --- a/ceph/qa/tasks/check_counter.py +++ b/ceph/qa/tasks/check_counter.py @@ -45,7 +45,7 @@ class CheckCounter(Task): targets = self.config.get('counters', {}) if cluster_name is None: - cluster_name = self.ctx.managers.keys()[0] + cluster_name = next(iter(self.ctx.managers.keys())) for daemon_type, counters in targets.items(): # List of 'a', 'b', 'c'... diff --git a/ceph/qa/tasks/cram.py b/ceph/qa/tasks/cram.py index a397eb51f..d06f09440 100644 --- a/ceph/qa/tasks/cram.py +++ b/ceph/qa/tasks/cram.py @@ -6,7 +6,7 @@ import os import six -from util.workunit import get_refspec_after_overrides +from tasks.util.workunit import get_refspec_after_overrides from teuthology import misc as teuthology from teuthology.parallel import parallel diff --git a/ceph/qa/tasks/devstack.py b/ceph/qa/tasks/devstack.py index 943a9ffff..9243071a4 100644 --- a/ceph/qa/tasks/devstack.py +++ b/ceph/qa/tasks/devstack.py @@ -1,9 +1,11 @@ #!/usr/bin/env python import contextlib import logging -from cStringIO import StringIO +from io import BytesIO import textwrap from configparser import ConfigParser + +import six import time from teuthology.orchestra import run @@ -58,8 +60,8 @@ def install(ctx, config): if not isinstance(config, dict): raise TypeError("config must be a dict") - devstack_node = ctx.cluster.only(is_devstack_node).remotes.keys()[0] - an_osd_node = ctx.cluster.only(is_osd_node).remotes.keys()[0] + devstack_node = next(iter(ctx.cluster.only(is_devstack_node).remotes.keys())) + an_osd_node = next(iter(ctx.cluster.only(is_osd_node).remotes.keys())) devstack_branch = config.get("branch", "master") install_devstack(devstack_node, devstack_branch) @@ -140,7 +142,7 @@ def distribute_ceph_keys(devstack_node, ceph_node): log.info("Copying Ceph keys to DevStack node...") def copy_key(from_remote, key_name, to_remote, dest_path, owner): - key_stringio = StringIO() + key_stringio = BytesIO() from_remote.run( args=['sudo', 'ceph', 'auth', 'get-or-create', key_name], stdout=key_stringio) @@ -172,14 +174,8 @@ def distribute_ceph_keys(devstack_node, ceph_node): def set_libvirt_secret(devstack_node, ceph_node): log.info("Setting libvirt secret...") - cinder_key_stringio = StringIO() - ceph_node.run(args=['sudo', 'ceph', 'auth', 'get-key', 'client.cinder'], - stdout=cinder_key_stringio) - cinder_key = cinder_key_stringio.getvalue().strip() - - uuid_stringio = StringIO() - devstack_node.run(args=['uuidgen'], stdout=uuid_stringio) - uuid = uuid_stringio.getvalue().strip() + cinder_key = ceph_node.sh('sudo ceph auth get-key client.cinder').strip() + uuid = devstack_node.sh('uuidgen').strip() secret_path = '/tmp/secret.xml' secret_template = textwrap.dedent(""" @@ -210,7 +206,7 @@ def update_devstack_config_files(devstack_node, secret_uuid): parser.read_file(config_stream) for (key, value) in update_dict.items(): parser.set(section, key, value) - out_stream = StringIO() + out_stream = six.StringIO() parser.write(out_stream) out_stream.seek(0) return out_stream @@ -254,8 +250,8 @@ def update_devstack_config_files(devstack_node, secret_uuid): for update in updates: file_name = update['name'] options = update['options'] - config_str = misc.get_file(devstack_node, file_name, sudo=True) - config_stream = StringIO(config_str) + config_data = misc.get_file(devstack_node, file_name, sudo=True) + config_stream = six.StringIO(config_data) backup_config(devstack_node, file_name) new_config_stream = update_config(file_name, config_stream, options) misc.sudo_write_file(devstack_node, file_name, new_config_stream) @@ -305,7 +301,7 @@ def exercise(ctx, config): if not isinstance(config, dict): raise TypeError("config must be a dict") - devstack_node = ctx.cluster.only(is_devstack_node).remotes.keys()[0] + devstack_node = next(iter(ctx.cluster.only(is_devstack_node).remotes.keys())) # TODO: save the log *and* preserve failures #devstack_archive_dir = create_devstack_archive(ctx, devstack_node) @@ -332,8 +328,8 @@ def create_devstack_archive(ctx, devstack_node): def smoke(ctx, config): log.info("Running a basic smoketest...") - devstack_node = ctx.cluster.only(is_devstack_node).remotes.keys()[0] - an_osd_node = ctx.cluster.only(is_osd_node).remotes.keys()[0] + devstack_node = next(iter(ctx.cluster.only(is_devstack_node).remotes.keys())) + an_osd_node = next(iter(ctx.cluster.only(is_osd_node).remotes.keys())) try: create_volume(devstack_node, an_osd_node, 'smoke0', 1) @@ -352,21 +348,17 @@ def create_volume(devstack_node, ceph_node, vol_name, size): size=size)) args = ['source', 'devstack/openrc', run.Raw('&&'), 'cinder', 'create', '--display-name', vol_name, size] - out_stream = StringIO() - devstack_node.run(args=args, stdout=out_stream, wait=True) - vol_info = parse_os_table(out_stream.getvalue()) + cinder_create = devstack_node.sh(args, wait=True) + vol_info = parse_os_table(cinder_create) log.debug("Volume info: %s", str(vol_info)) - out_stream = StringIO() try: - ceph_node.run(args="rbd --id cinder ls -l volumes", stdout=out_stream, - wait=True) + rbd_output = ceph_node.sh("rbd --id cinder ls -l volumes", wait=True) except run.CommandFailedError: log.debug("Original rbd call failed; retrying without '--id cinder'") - ceph_node.run(args="rbd ls -l volumes", stdout=out_stream, - wait=True) + rbd_output = ceph_node.sh("rbd ls -l volumes", wait=True) - assert vol_info['id'] in out_stream.getvalue(), \ + assert vol_info['id'] in rbd_output, \ "Volume not found on Ceph cluster" assert vol_info['size'] == size, \ "Volume size on Ceph cluster is different than specified" diff --git a/ceph/qa/tasks/die_on_err.py b/ceph/qa/tasks/die_on_err.py index ee157f4af..a6aa4c632 100644 --- a/ceph/qa/tasks/die_on_err.py +++ b/ceph/qa/tasks/die_on_err.py @@ -6,7 +6,7 @@ import logging import time from teuthology.orchestra import run -import ceph_manager +from tasks import ceph_manager from teuthology import misc as teuthology log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/divergent_priors.py b/ceph/qa/tasks/divergent_priors.py index b565c774c..e000bb2bb 100644 --- a/ceph/qa/tasks/divergent_priors.py +++ b/ceph/qa/tasks/divergent_priors.py @@ -5,7 +5,7 @@ import logging import time from teuthology import misc as teuthology -from util.rados import rados +from tasks.util.rados import rados log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/divergent_priors2.py b/ceph/qa/tasks/divergent_priors2.py index dda358b04..4d4b07fc4 100644 --- a/ceph/qa/tasks/divergent_priors2.py +++ b/ceph/qa/tasks/divergent_priors2.py @@ -6,7 +6,7 @@ import time from teuthology.exceptions import CommandFailedError from teuthology import misc as teuthology -from util.rados import rados +from tasks.util.rados import rados import os diff --git a/ceph/qa/tasks/dnsmasq.py b/ceph/qa/tasks/dnsmasq.py index 2bf3feaf0..352ed246b 100644 --- a/ceph/qa/tasks/dnsmasq.py +++ b/ceph/qa/tasks/dnsmasq.py @@ -8,7 +8,7 @@ from teuthology import misc from teuthology.exceptions import ConfigError from teuthology import contextutil from teuthology import packaging -from util import get_remote_for_role +from tasks.util import get_remote_for_role log = logging.getLogger(__name__) @@ -83,7 +83,7 @@ def setup_dnsmasq(remote, testdir, cnames): # restart dnsmasq remote.run(args=['sudo', 'systemctl', 'restart', 'dnsmasq']) # verify dns name is set - remote.run(args=['ping', '-c', '4', cnames.keys()[0]]) + remote.run(args=['ping', '-c', '4', next(iter(cnames.keys()))]) try: yield diff --git a/ceph/qa/tasks/dump_stuck.py b/ceph/qa/tasks/dump_stuck.py index 237d9127f..4971f1916 100644 --- a/ceph/qa/tasks/dump_stuck.py +++ b/ceph/qa/tasks/dump_stuck.py @@ -4,7 +4,7 @@ Dump_stuck command import logging import time -import ceph_manager +from tasks import ceph_manager from teuthology import misc as teuthology diff --git a/ceph/qa/tasks/ec_lost_unfound.py b/ceph/qa/tasks/ec_lost_unfound.py index 2360ea92b..e12b6901c 100644 --- a/ceph/qa/tasks/ec_lost_unfound.py +++ b/ceph/qa/tasks/ec_lost_unfound.py @@ -1,12 +1,12 @@ """ Lost_unfound """ -from teuthology.orchestra import run import logging -import ceph_manager -from teuthology import misc as teuthology -from util.rados import rados import time +from tasks import ceph_manager +from tasks.util.rados import rados +from teuthology import misc as teuthology +from teuthology.orchestra import run log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/filestore_idempotent.py b/ceph/qa/tasks/filestore_idempotent.py index 01b562905..319bef768 100644 --- a/ceph/qa/tasks/filestore_idempotent.py +++ b/ceph/qa/tasks/filestore_idempotent.py @@ -31,7 +31,7 @@ def task(ctx, config): clients = config.keys() # just use the first client... - client = clients[0]; + client = next(iter(clients)) (remote,) = ctx.cluster.only(client).remotes.keys() testdir = teuthology.get_testdir(ctx) diff --git a/ceph/qa/tasks/fs.py b/ceph/qa/tasks/fs.py index 4b47e754b..ca84dc7a5 100644 --- a/ceph/qa/tasks/fs.py +++ b/ceph/qa/tasks/fs.py @@ -4,6 +4,7 @@ CephFS sub-tasks. import logging import re +import six from tasks.cephfs.filesystem import Filesystem @@ -38,7 +39,7 @@ def clients_evicted(ctx, config): for rank in fs.get_ranks(status=status): ls = fs.rank_asok(['session', 'ls'], rank=rank['rank'], status=status) for session in ls: - for client, evicted in clients.viewitems(): + for client, evicted in six.viewitems(clients): mount = mounts.get(client) if mount is not None: global_id = mount.get_global_id() @@ -51,7 +52,7 @@ def clients_evicted(ctx, config): no_session = set(clients) - has_session should_assert = False - for client, evicted in clients.viewitems(): + for client, evicted in six.viewitems(clients): mount = mounts.get(client) if mount is not None: if evicted: diff --git a/ceph/qa/tasks/kclient.py b/ceph/qa/tasks/kclient.py index 88f27366a..ce0d73f56 100644 --- a/ceph/qa/tasks/kclient.py +++ b/ceph/qa/tasks/kclient.py @@ -8,7 +8,7 @@ from teuthology.misc import deep_merge from teuthology.orchestra.run import CommandFailedError from teuthology import misc from teuthology.contextutil import MaxWhileTries -from cephfs.kernel_mount import KernelMount +from tasks.cephfs.kernel_mount import KernelMount log = logging.getLogger(__name__) @@ -109,7 +109,7 @@ def task(ctx, config): try: mount.umount() except (CommandFailedError, MaxWhileTries): - log.warn("Ordinary umount failed, forcing...") + log.warning("Ordinary umount failed, forcing...") forced = True mount.umount_wait(force=True) diff --git a/ceph/qa/tasks/keystone.py b/ceph/qa/tasks/keystone.py index 4433ce204..bcb100602 100644 --- a/ceph/qa/tasks/keystone.py +++ b/ceph/qa/tasks/keystone.py @@ -4,7 +4,10 @@ Deploy and configure Keystone for Teuthology import argparse import contextlib import logging -from cStringIO import StringIO + +# still need this for python3.6 +from collections import OrderedDict +from itertools import chain from teuthology import misc as teuthology from teuthology import contextutil @@ -28,11 +31,9 @@ def run_in_keystone_dir(ctx, client, args, **kwargs): def get_toxvenv_dir(ctx): return ctx.tox.venv_path -def run_in_tox_venv(ctx, remote, args, **kwargs): - return remote.run( - args=[ 'source', '{}/bin/activate'.format(get_toxvenv_dir(ctx)), run.Raw('&&') ] + args, - **kwargs - ) +def toxvenv_sh(ctx, remote, args, **kwargs): + activate = get_toxvenv_dir(ctx) + '/bin/activate' + return remote.sh(['source', activate, run.Raw('&&')] + args, **kwargs) def run_in_keystone_venv(ctx, client, args): run_in_keystone_dir(ctx, client, @@ -107,12 +108,10 @@ def install_packages(ctx, config): for (client, _) in config.items(): (remote,) = ctx.cluster.only(client).remotes.keys() # use bindep to read which dependencies we need from keystone/bindep.txt - run_in_tox_venv(ctx, remote, ['pip', 'install', 'bindep']) - r = run_in_tox_venv(ctx, remote, + toxvenv_sh(ctx, remote, ['pip', 'install', 'bindep']) + packages[client] = toxvenv_sh(ctx, remote, ['bindep', '--brief', '--file', '{}/bindep.txt'.format(get_keystone_dir(ctx))], - stdout=StringIO(), - check_status=False) # returns 1 on success? - packages[client] = r.stdout.getvalue().splitlines() + check_status=False).splitlines() # returns 1 on success? for dep in packages[client]: install_package(dep, remote) try: @@ -141,9 +140,7 @@ def setup_venv(ctx, config): ]) run_in_keystone_venv(ctx, client, - [ 'pip', 'install', 'python-openstackclient<=3.19.0', - '-r', 'requirements.txt' - ]) + [ 'pip', 'install', 'python-openstackclient']) try: yield finally: @@ -159,15 +156,16 @@ def configure_instance(ctx, config): # prepare the config file run_in_keystone_dir(ctx, client, [ - 'cp', '-f', - 'etc/keystone.conf.sample', - 'etc/keystone.conf' + 'source', + f'{get_toxvenv_dir(ctx)}/bin/activate', + run.Raw('&&'), + 'tox', '-e', 'genconfig' ]) run_in_keystone_dir(ctx, client, [ - 'sed', - '-e', 's/#admin_token =.*/admin_token = ADMIN/', - '-i', 'etc/keystone.conf' + 'cp', '-f', + 'etc/keystone.conf.sample', + 'etc/keystone.conf' ]) run_in_keystone_dir(ctx, client, [ @@ -265,77 +263,96 @@ def run_keystone(ctx, config): cluster_name).stop() -def dict_to_args(special, items): +def dict_to_args(specials, items): """ Transform [(key1, val1), (special, val_special), (key3, val3) ] into: [ '--key1', 'val1', '--key3', 'val3', 'val_special' ] """ - args=[] + args = [] + special_vals = OrderedDict((k, '') for k in specials.split(',')) for (k, v) in items: - if k == special: - special_val = v + if k in special_vals: + special_vals[k] = v else: args.append('--{k}'.format(k=k)) args.append(v) - if special_val: - args.append(special_val) + args.extend(arg for arg in special_vals.values() if arg) return args -def run_section_cmds(ctx, cclient, section_cmd, special, +def run_section_cmds(ctx, cclient, section_cmd, specials, section_config_list): admin_host, admin_port = ctx.keystone.admin_endpoints[cclient] auth_section = [ - ( 'os-token', 'ADMIN' ), - ( 'os-identity-api-version', '2.0' ), - ( 'os-url', 'http://{host}:{port}/v2.0'.format(host=admin_host, - port=admin_port) ), + ( 'os-username', 'admin' ), + ( 'os-password', 'ADMIN' ), + ( 'os-user-domain-id', 'default' ), + ( 'os-project-name', 'admin' ), + ( 'os-project-domain-id', 'default' ), + ( 'os-identity-api-version', '3' ), + ( 'os-auth-url', 'http://{host}:{port}/v3'.format(host=admin_host, + port=admin_port) ), ] for section_item in section_config_list: run_in_keystone_venv(ctx, cclient, [ 'openstack' ] + section_cmd.split() + - dict_to_args(special, auth_section + section_item.items()) + + dict_to_args(specials, auth_section + list(section_item.items())) + [ '--debug' ]) def create_endpoint(ctx, cclient, service, url, adminurl=None): - endpoint_section = { - 'service': service, - 'publicurl': url, - } + endpoint_sections = [ + {'service': service, 'interface': 'public', 'url': url}, + ] if adminurl: - endpoint_section.update( { - 'adminurl': adminurl, - } ) - return run_section_cmds(ctx, cclient, 'endpoint create', 'service', - [ endpoint_section ]) + endpoint_sections.append( + {'service': service, 'interface': 'admin', 'url': adminurl} + ) + run_section_cmds(ctx, cclient, 'endpoint create', + 'service,interface,url', + endpoint_sections) @contextlib.contextmanager def fill_keystone(ctx, config): assert isinstance(config, dict) for (cclient, cconfig) in config.items(): + public_host, public_port = ctx.keystone.public_endpoints[cclient] + url = 'http://{host}:{port}/v3'.format(host=public_host, + port=public_port) + admin_host, admin_port = ctx.keystone.admin_endpoints[cclient] + admin_url = 'http://{host}:{port}/v3'.format(host=admin_host, + port=admin_port) + opts = {'password': 'ADMIN', + 'username': 'admin', + 'project-name': 'admin', + 'role-name': 'admin', + 'service-name': 'keystone', + 'region-id': 'RegionOne', + 'admin-url': admin_url, + 'public-url': url} + bootstrap_args = chain.from_iterable(('--bootstrap-{}'.format(k), v) + for k, v in opts.items()) + run_in_keystone_venv(ctx, cclient, + ['keystone-manage', 'bootstrap'] + + list(bootstrap_args)) + # configure tenants/projects + run_section_cmds(ctx, cclient, 'domain create', 'name', + cconfig.get('domains', [])) run_section_cmds(ctx, cclient, 'project create', 'name', - cconfig['tenants']) + cconfig.get('projects', [])) run_section_cmds(ctx, cclient, 'user create', 'name', - cconfig['users']) + cconfig.get('users', [])) run_section_cmds(ctx, cclient, 'role create', 'name', - cconfig['roles']) + cconfig.get('roles', [])) run_section_cmds(ctx, cclient, 'role add', 'name', - cconfig['role-mappings']) - run_section_cmds(ctx, cclient, 'service create', 'name', - cconfig['services']) + cconfig.get('role-mappings', [])) + run_section_cmds(ctx, cclient, 'service create', 'type', + cconfig.get('services', [])) - public_host, public_port = ctx.keystone.public_endpoints[cclient] - url = 'http://{host}:{port}/v2.0'.format(host=public_host, - port=public_port) - admin_host, admin_port = ctx.keystone.admin_endpoints[cclient] - admin_url = 'http://{host}:{port}/v2.0'.format(host=admin_host, - port=admin_port) - create_endpoint(ctx, cclient, 'keystone', url, admin_url) # for the deferred endpoint creation; currently it's used in rgw.py ctx.keystone.create_endpoint = create_endpoint @@ -373,7 +390,10 @@ def task(ctx, config): - keystone: client.0: force-branch: master - tenants: + domains: + - name: default + description: Default Domain + projects: - name: admin description: Admin Tenant users: diff --git a/ceph/qa/tasks/lost_unfound.py b/ceph/qa/tasks/lost_unfound.py index d51b96693..ab17a95dc 100644 --- a/ceph/qa/tasks/lost_unfound.py +++ b/ceph/qa/tasks/lost_unfound.py @@ -3,10 +3,10 @@ Lost_unfound """ import logging import time -import ceph_manager +from tasks import ceph_manager +from tasks.util.rados import rados from teuthology import misc as teuthology from teuthology.orchestra import run -from util.rados import rados log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/mds_creation_failure.py b/ceph/qa/tasks/mds_creation_failure.py index 29e2c3513..58314086c 100644 --- a/ceph/qa/tasks/mds_creation_failure.py +++ b/ceph/qa/tasks/mds_creation_failure.py @@ -3,7 +3,7 @@ import logging import contextlib import time -import ceph_manager +from tasks import ceph_manager from teuthology import misc from teuthology.orchestra.run import CommandFailedError, Raw diff --git a/ceph/qa/tasks/mds_thrash.py b/ceph/qa/tasks/mds_thrash.py index 07ec039ed..11e647b8a 100644 --- a/ceph/qa/tasks/mds_thrash.py +++ b/ceph/qa/tasks/mds_thrash.py @@ -3,7 +3,6 @@ Thrash mds by simulating failures """ import logging import contextlib -import ceph_manager import itertools import random import time @@ -13,6 +12,7 @@ from gevent.greenlet import Greenlet from gevent.event import Event from teuthology import misc as teuthology +from tasks import ceph_manager from tasks.cephfs.filesystem import MDSCluster, Filesystem from tasks.thrasher import Thrasher @@ -186,10 +186,11 @@ class MDSThrasher(Thrasher, Greenlet): status = self.fs.status() max_mds = status.get_fsmap(self.fs.id)['mdsmap']['max_mds'] ranks = list(status.get_ranks(self.fs.id)) - stopping = filter(lambda info: "up:stopping" == info['state'], ranks) - actives = filter(lambda info: "up:active" == info['state'] and "laggy_since" not in info, ranks) + stopping = sum(1 for _ in ranks if "up:stopping" == _['state']) + actives = sum(1 for _ in ranks + if "up:active" == _['state'] and "laggy_since" not in _) - if not bool(self.config.get('thrash_while_stopping', False)) and len(stopping) > 0: + if not bool(self.config.get('thrash_while_stopping', False)) and stopping > 0: if itercount % 5 == 0: self.log('cluster is considered unstable while MDS are in up:stopping (!thrash_while_stopping)') else: @@ -201,14 +202,14 @@ class MDSThrasher(Thrasher, Greenlet): return status except: pass # no rank present - if len(actives) >= max_mds: + if actives >= max_mds: # no replacement can occur! self.log("cluster has {actives} actives (max_mds is {max_mds}), no MDS can replace rank {rank}".format( - actives=len(actives), max_mds=max_mds, rank=rank)) + actives=actives, max_mds=max_mds, rank=rank)) return status else: - if len(actives) == max_mds: - self.log('mds cluster has {count} alive and active, now stable!'.format(count = len(actives))) + if actives == max_mds: + self.log('mds cluster has {count} alive and active, now stable!'.format(count = actives)) return status, None if itercount > 300/2: # 5 minutes raise RuntimeError('timeout waiting for cluster to stabilize') @@ -245,7 +246,7 @@ class MDSThrasher(Thrasher, Greenlet): if random.random() <= self.thrash_max_mds: max_mds = status.get_fsmap(self.fs.id)['mdsmap']['max_mds'] - options = range(1, max_mds)+range(max_mds+1, self.max_mds+1) + options = list(range(1, max_mds))+list(range(max_mds+1, self.max_mds+1)) if len(options) > 0: sample = random.sample(options, 1) new_max_mds = sample[0] diff --git a/ceph/qa/tasks/mgr/dashboard/helper.py b/ceph/qa/tasks/mgr/dashboard/helper.py index 6419d318a..1acca1537 100644 --- a/ceph/qa/tasks/mgr/dashboard/helper.py +++ b/ceph/qa/tasks/mgr/dashboard/helper.py @@ -11,7 +11,7 @@ import requests import six from teuthology.exceptions import CommandFailedError -from ..mgr_test_case import MgrTestCase +from tasks.mgr.mgr_test_case import MgrTestCase log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/mgr/dashboard/test_auth.py b/ceph/qa/tasks/mgr/dashboard/test_auth.py index e76708a9c..468fe3796 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_auth.py +++ b/ceph/qa/tasks/mgr/dashboard/test_auth.py @@ -6,7 +6,7 @@ import time import jwt -from .helper import DashboardTestCase, JObj, JLeaf +from tasks.mgr.dashboard.helper import DashboardTestCase, JObj, JLeaf class AuthTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_cephfs.py b/ceph/qa/tasks/mgr/dashboard/test_cephfs.py index e6594450f..291d4d85c 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_cephfs.py +++ b/ceph/qa/tasks/mgr/dashboard/test_cephfs.py @@ -4,7 +4,7 @@ from __future__ import absolute_import import six from contextlib import contextmanager -from .helper import DashboardTestCase, JObj, JList, JLeaf +from tasks.mgr.dashboard.helper import DashboardTestCase, JObj, JList, JLeaf class CephfsTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_cluster_configuration.py b/ceph/qa/tasks/mgr/dashboard/test_cluster_configuration.py index 61d18000a..9f134cd87 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_cluster_configuration.py +++ b/ceph/qa/tasks/mgr/dashboard/test_cluster_configuration.py @@ -2,7 +2,7 @@ from __future__ import absolute_import import time -from .helper import DashboardTestCase +from tasks.mgr.dashboard.helper import DashboardTestCase class ClusterConfigurationTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_crush_rule.py b/ceph/qa/tasks/mgr/dashboard/test_crush_rule.py index a0bca63ff..33949925b 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_crush_rule.py +++ b/ceph/qa/tasks/mgr/dashboard/test_crush_rule.py @@ -4,7 +4,7 @@ from __future__ import absolute_import import six -from .helper import DashboardTestCase, JObj, JList +from tasks.mgr.dashboard.helper import DashboardTestCase, JObj, JList class CrushRuleTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_erasure_code_profile.py b/ceph/qa/tasks/mgr/dashboard/test_erasure_code_profile.py index 111e37c7e..c6c829577 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_erasure_code_profile.py +++ b/ceph/qa/tasks/mgr/dashboard/test_erasure_code_profile.py @@ -4,7 +4,7 @@ from __future__ import absolute_import import six -from .helper import DashboardTestCase, JObj, JList +from tasks.mgr.dashboard.helper import DashboardTestCase, JObj, JList class ECPTest(DashboardTestCase): @@ -102,9 +102,8 @@ class ECPTest(DashboardTestCase): self._get('/ui-api/erasure_code_profile/info') self.assertSchemaBody(JObj({ 'names': JList(six.string_types), - 'failure_domains': JList(six.string_types), 'plugins': JList(six.string_types), - 'devices': JList(six.string_types), 'directory': six.string_types, + 'nodes': JList(JObj({}, allow_unknown=True)) })) diff --git a/ceph/qa/tasks/mgr/dashboard/test_ganesha.py b/ceph/qa/tasks/mgr/dashboard/test_ganesha.py index cd869a00e..0311dadda 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_ganesha.py +++ b/ceph/qa/tasks/mgr/dashboard/test_ganesha.py @@ -4,7 +4,7 @@ from __future__ import absolute_import -from .helper import DashboardTestCase +from tasks.mgr.dashboard.helper import DashboardTestCase class GaneshaTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_health.py b/ceph/qa/tasks/mgr/dashboard/test_health.py index 08551e488..e7bfb4fab 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_health.py +++ b/ceph/qa/tasks/mgr/dashboard/test_health.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -from .helper import DashboardTestCase, JAny, JLeaf, JList, JObj +from tasks.mgr.dashboard.helper import DashboardTestCase, JAny, JLeaf, JList, JObj class HealthTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_host.py b/ceph/qa/tasks/mgr/dashboard/test_host.py index d608dc247..407207133 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_host.py +++ b/ceph/qa/tasks/mgr/dashboard/test_host.py @@ -2,8 +2,8 @@ from __future__ import absolute_import import json -from .helper import DashboardTestCase, JList, JObj -from .test_orchestrator import test_data +from tasks.mgr.dashboard.helper import DashboardTestCase, JList, JObj +from tasks.mgr.dashboard.test_orchestrator import test_data class HostControllerTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_logs.py b/ceph/qa/tasks/mgr/dashboard/test_logs.py index 17d5d830c..5108161ad 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_logs.py +++ b/ceph/qa/tasks/mgr/dashboard/test_logs.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -from .helper import DashboardTestCase, JList, JObj +from tasks.mgr.dashboard.helper import DashboardTestCase, JList, JObj class LogsTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_mgr_module.py b/ceph/qa/tasks/mgr/dashboard/test_mgr_module.py index 080b8b64c..ec6dbb47f 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_mgr_module.py +++ b/ceph/qa/tasks/mgr/dashboard/test_mgr_module.py @@ -4,7 +4,7 @@ from __future__ import absolute_import import logging import requests -from .helper import DashboardTestCase, JAny, JObj, JList, JLeaf +from tasks.mgr.dashboard.helper import DashboardTestCase, JAny, JObj, JList, JLeaf logger = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/mgr/dashboard/test_monitor.py b/ceph/qa/tasks/mgr/dashboard/test_monitor.py index 0cf7e25a2..1558cdc82 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_monitor.py +++ b/ceph/qa/tasks/mgr/dashboard/test_monitor.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -from .helper import DashboardTestCase +from tasks.mgr.dashboard.helper import DashboardTestCase class MonitorTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_orchestrator.py b/ceph/qa/tasks/mgr/dashboard/test_orchestrator.py index 9f4204379..4f248a1c3 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_orchestrator.py +++ b/ceph/qa/tasks/mgr/dashboard/test_orchestrator.py @@ -2,7 +2,7 @@ from __future__ import absolute_import import json -from .helper import DashboardTestCase +from tasks.mgr.dashboard.helper import DashboardTestCase test_data = { diff --git a/ceph/qa/tasks/mgr/dashboard/test_osd.py b/ceph/qa/tasks/mgr/dashboard/test_osd.py index 1bd75e4b0..4f2028d22 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_osd.py +++ b/ceph/qa/tasks/mgr/dashboard/test_osd.py @@ -4,7 +4,7 @@ from __future__ import absolute_import import json -from .helper import DashboardTestCase, JObj, JAny, JList, JLeaf, JTuple +from tasks.mgr.dashboard.helper import DashboardTestCase, JObj, JAny, JList, JLeaf, JTuple class OsdTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_perf_counters.py b/ceph/qa/tasks/mgr/dashboard/test_perf_counters.py index c01368bce..99133a77c 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_perf_counters.py +++ b/ceph/qa/tasks/mgr/dashboard/test_perf_counters.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -from .helper import DashboardTestCase, JObj +from tasks.mgr.dashboard.helper import DashboardTestCase, JObj class PerfCountersControllerTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_pool.py b/ceph/qa/tasks/mgr/dashboard/test_pool.py index 969318d2a..bf40ac206 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_pool.py +++ b/ceph/qa/tasks/mgr/dashboard/test_pool.py @@ -6,7 +6,7 @@ import six import time from contextlib import contextmanager -from .helper import DashboardTestCase, JAny, JList, JObj +from tasks.mgr.dashboard.helper import DashboardTestCase, JAny, JList, JObj log = logging.getLogger(__name__) @@ -414,4 +414,5 @@ class PoolTest(DashboardTestCase): 'pg_autoscale_modes': JList(six.string_types), 'erasure_code_profiles': JList(JObj({}, allow_unknown=True)), 'used_rules': JObj({}, allow_unknown=True), + 'used_profiles': JObj({}, allow_unknown=True), })) diff --git a/ceph/qa/tasks/mgr/dashboard/test_rbd.py b/ceph/qa/tasks/mgr/dashboard/test_rbd.py index 108b2f033..1c89651b4 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_rbd.py +++ b/ceph/qa/tasks/mgr/dashboard/test_rbd.py @@ -5,7 +5,7 @@ from __future__ import absolute_import import time -from .helper import DashboardTestCase, JObj, JLeaf, JList +from tasks.mgr.dashboard.helper import DashboardTestCase, JObj, JLeaf, JList class RbdTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_rbd_mirroring.py b/ceph/qa/tasks/mgr/dashboard/test_rbd_mirroring.py index 39e5f895f..f8268f352 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_rbd_mirroring.py +++ b/ceph/qa/tasks/mgr/dashboard/test_rbd_mirroring.py @@ -3,7 +3,7 @@ from __future__ import absolute_import -from .helper import DashboardTestCase +from tasks.mgr.dashboard.helper import DashboardTestCase class RbdMirroringTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_requests.py b/ceph/qa/tasks/mgr/dashboard/test_requests.py index 0d9f8d9ba..22376c0a2 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_requests.py +++ b/ceph/qa/tasks/mgr/dashboard/test_requests.py @@ -2,7 +2,7 @@ from __future__ import absolute_import -from .helper import DashboardTestCase +from tasks.mgr.dashboard.helper import DashboardTestCase class RequestsTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_rgw.py b/ceph/qa/tasks/mgr/dashboard/test_rgw.py index 47a6de245..3e5b73599 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_rgw.py +++ b/ceph/qa/tasks/mgr/dashboard/test_rgw.py @@ -4,13 +4,13 @@ from __future__ import absolute_import import base64 import logging import time -import urllib from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.twofactor.totp import TOTP from cryptography.hazmat.primitives.hashes import SHA1 +from six.moves.urllib import parse -from .helper import DashboardTestCase, JObj, JList, JLeaf +from tasks.mgr.dashboard.helper import DashboardTestCase, JObj, JList, JLeaf logger = logging.getLogger(__name__) @@ -112,6 +112,27 @@ class RgwApiCredentialsTest(RgwTestCase): data['message']) +class RgwSiteTest(RgwTestCase): + + AUTH_ROLES = ['rgw-manager'] + + def test_get_placement_targets(self): + data = self._get('/api/rgw/site?query=placement-targets') + self.assertStatus(200) + self.assertSchema(data, JObj({ + 'zonegroup': str, + 'placement_targets': JList(JObj({ + 'name': str, + 'data_pool': str + })) + })) + + def test_get_realms(self): + data = self._get('/api/rgw/site?query=realms') + self.assertStatus(200) + self.assertSchema(data, JList(str)) + + class RgwBucketTest(RgwTestCase): _mfa_token_serial = '1' @@ -292,7 +313,7 @@ class RgwBucketTest(RgwTestCase): def _verify_tenant_bucket(bucket, tenant, uid): full_bucket_name = '{}/{}'.format(tenant, bucket) _data = self._get('/api/rgw/bucket/{}'.format( - urllib.quote_plus(full_bucket_name))) + parse.quote_plus(full_bucket_name))) self.assertStatus(200) self.assertSchema(_data, JObj(sub_elems={ 'owner': JLeaf(str), @@ -314,7 +335,7 @@ class RgwBucketTest(RgwTestCase): # Update bucket: different user with different tenant, enable versioning. self._put( '/api/rgw/bucket/{}'.format( - urllib.quote_plus('testx/teuth-test-bucket')), + parse.quote_plus('testx/teuth-test-bucket')), params={ 'bucket_id': data['id'], 'uid': 'testx2$teuth-test-user2', @@ -326,7 +347,7 @@ class RgwBucketTest(RgwTestCase): # Change owner to a non-tenanted user self._put( '/api/rgw/bucket/{}'.format( - urllib.quote_plus('testx2/teuth-test-bucket')), + parse.quote_plus('testx2/teuth-test-bucket')), params={ 'bucket_id': data['id'], 'uid': 'admin' @@ -355,7 +376,7 @@ class RgwBucketTest(RgwTestCase): # Delete the bucket. self._delete('/api/rgw/bucket/{}'.format( - urllib.quote_plus('testx/teuth-test-bucket'))) + parse.quote_plus('testx/teuth-test-bucket'))) self.assertStatus(204) data = self._get('/api/rgw/bucket') self.assertStatus(200) diff --git a/ceph/qa/tasks/mgr/dashboard/test_role.py b/ceph/qa/tasks/mgr/dashboard/test_role.py index dbfaea9e4..d678fa195 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_role.py +++ b/ceph/qa/tasks/mgr/dashboard/test_role.py @@ -2,7 +2,7 @@ from __future__ import absolute_import -from .helper import DashboardTestCase +from tasks.mgr.dashboard.helper import DashboardTestCase class RoleTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_settings.py b/ceph/qa/tasks/mgr/dashboard/test_settings.py index 2d890484a..4c6ebeaca 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_settings.py +++ b/ceph/qa/tasks/mgr/dashboard/test_settings.py @@ -2,7 +2,7 @@ from __future__ import absolute_import -from .helper import DashboardTestCase, JList, JObj, JAny +from tasks.mgr.dashboard.helper import DashboardTestCase, JList, JObj, JAny class SettingsTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_summary.py b/ceph/qa/tasks/mgr/dashboard/test_summary.py index a31f89146..808107a31 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_summary.py +++ b/ceph/qa/tasks/mgr/dashboard/test_summary.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -from .helper import DashboardTestCase +from tasks.mgr.dashboard.helper import DashboardTestCase class SummaryTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/dashboard/test_telemetry.py b/ceph/qa/tasks/mgr/dashboard/test_telemetry.py new file mode 100644 index 000000000..840ccddca --- /dev/null +++ b/ceph/qa/tasks/mgr/dashboard/test_telemetry.py @@ -0,0 +1,89 @@ +from .helper import DashboardTestCase, JObj + + +class TelemetryTest(DashboardTestCase): + + pre_enabled_status = True + + @classmethod + def setUpClass(cls): + super(TelemetryTest, cls).setUpClass() + data = cls._get('/api/mgr/module/telemetry') + cls.pre_enabled_status = data['enabled'] + + @classmethod + def tearDownClass(cls): + if cls.pre_enabled_status: + cls._enable_module() + else: + cls._disable_module() + super(TelemetryTest, cls).tearDownClass() + + def test_disable_module(self): + self._enable_module() + self._check_telemetry_enabled(True) + self._disable_module() + self._check_telemetry_enabled(False) + + def test_enable_module_correct_license(self): + self._disable_module() + self._check_telemetry_enabled(False) + + self._put('/api/telemetry', { + 'enable': True, + 'license_name': 'sharing-1-0' + }) + self.assertStatus(200) + self._check_telemetry_enabled(True) + + def test_enable_module_empty_license(self): + self._disable_module() + self._check_telemetry_enabled(False) + + self._put('/api/telemetry', { + 'enable': True, + 'license_name': '' + }) + self.assertStatus(400) + self.assertError(code='telemetry_enable_license_missing') + self._check_telemetry_enabled(False) + + def test_enable_module_invalid_license(self): + self._disable_module() + self._check_telemetry_enabled(False) + + self._put('/api/telemetry', { + 'enable': True, + 'license_name': 'invalid-license' + }) + self.assertStatus(400) + self.assertError(code='telemetry_enable_license_missing') + self._check_telemetry_enabled(False) + + def test_get_report(self): + self._enable_module() + data = self._get('/api/telemetry/report') + self.assertStatus(200) + schema = JObj({ + 'report': JObj({}, allow_unknown=True), + 'device_report': JObj({}, allow_unknown=True) + }) + self.assertSchema(data, schema) + + @classmethod + def _enable_module(cls): + cls._put('/api/telemetry', { + 'enable': True, + 'license_name': 'sharing-1-0' + }) + + @classmethod + def _disable_module(cls): + cls._put('/api/telemetry', { + 'enable': False + }) + + def _check_telemetry_enabled(self, enabled): + data = self._get('/api/mgr/module/telemetry') + self.assertStatus(200) + self.assertEqual(data['enabled'], enabled) diff --git a/ceph/qa/tasks/mgr/dashboard/test_user.py b/ceph/qa/tasks/mgr/dashboard/test_user.py index ea7beee6d..f3606ed85 100644 --- a/ceph/qa/tasks/mgr/dashboard/test_user.py +++ b/ceph/qa/tasks/mgr/dashboard/test_user.py @@ -6,7 +6,7 @@ import time from datetime import datetime, timedelta -from .helper import DashboardTestCase, JObj, JLeaf +from tasks.mgr.dashboard.helper import DashboardTestCase, JObj, JLeaf class UserTest(DashboardTestCase): diff --git a/ceph/qa/tasks/mgr/mgr_test_case.py b/ceph/qa/tasks/mgr/mgr_test_case.py index 70faa2a2b..37baeb20e 100644 --- a/ceph/qa/tasks/mgr/mgr_test_case.py +++ b/ceph/qa/tasks/mgr/mgr_test_case.py @@ -73,7 +73,7 @@ class MgrTestCase(CephTestCase): # Unload all non-default plugins loaded = json.loads(cls.mgr_cluster.mon_manager.raw_cluster_cmd( "mgr", "module", "ls"))['enabled_modules'] - unload_modules = set(loaded) - {"restful"} + unload_modules = set(loaded) - {"cephadm", "restful"} for m in unload_modules: cls.mgr_cluster.mon_manager.raw_cluster_cmd( diff --git a/ceph/qa/tasks/mgr/test_crash.py b/ceph/qa/tasks/mgr/test_crash.py index 149221e9f..e73f19890 100644 --- a/ceph/qa/tasks/mgr/test_crash.py +++ b/ceph/qa/tasks/mgr/test_crash.py @@ -1,6 +1,6 @@ -from mgr_test_case import MgrTestCase +from tasks.mgr.mgr_test_case import MgrTestCase import json import logging @@ -74,7 +74,7 @@ class TestCrash(MgrTestCase): self.assertIn(crash['crash_id'], retstr) def test_rm(self): - crashid = self.crashes.keys()[0] + crashid = next(iter(self.crashes.keys())) self.assertEqual( 0, self.mgr_cluster.mon_manager.raw_cluster_cmd_result( diff --git a/ceph/qa/tasks/mgr/test_dashboard.py b/ceph/qa/tasks/mgr/test_dashboard.py index 783c492d3..44b6602a6 100644 --- a/ceph/qa/tasks/mgr/test_dashboard.py +++ b/ceph/qa/tasks/mgr/test_dashboard.py @@ -1,6 +1,6 @@ -from mgr_test_case import MgrTestCase +from tasks.mgr.mgr_test_case import MgrTestCase import logging import requests diff --git a/ceph/qa/tasks/mgr/test_insights.py b/ceph/qa/tasks/mgr/test_insights.py index c483e3abf..423a4a8fe 100644 --- a/ceph/qa/tasks/mgr/test_insights.py +++ b/ceph/qa/tasks/mgr/test_insights.py @@ -2,7 +2,8 @@ import logging import json import datetime import time -from mgr_test_case import MgrTestCase + +from tasks.mgr.mgr_test_case import MgrTestCase log = logging.getLogger(__name__) UUID = 'd5775432-0742-44a3-a435-45095e32e6b2' diff --git a/ceph/qa/tasks/mgr/test_orchestrator_cli.py b/ceph/qa/tasks/mgr/test_orchestrator_cli.py index 5232af299..db9226323 100644 --- a/ceph/qa/tasks/mgr/test_orchestrator_cli.py +++ b/ceph/qa/tasks/mgr/test_orchestrator_cli.py @@ -5,7 +5,7 @@ from time import sleep from teuthology.exceptions import CommandFailedError -from mgr_test_case import MgrTestCase +from tasks.mgr.mgr_test_case import MgrTestCase log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/mgr/test_progress.py b/ceph/qa/tasks/mgr/test_progress.py index 18460402e..40cd3a0ca 100644 --- a/ceph/qa/tasks/mgr/test_progress.py +++ b/ceph/qa/tasks/mgr/test_progress.py @@ -3,7 +3,7 @@ import json import logging import time -from mgr_test_case import MgrTestCase +from tasks.mgr.mgr_test_case import MgrTestCase log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/mgr/test_prometheus.py b/ceph/qa/tasks/mgr/test_prometheus.py index f91487174..867d5cd5d 100644 --- a/ceph/qa/tasks/mgr/test_prometheus.py +++ b/ceph/qa/tasks/mgr/test_prometheus.py @@ -1,6 +1,4 @@ - - -from mgr_test_case import MgrTestCase +from tasks.mgr.mgr_test_case import MgrTestCase import json import logging diff --git a/ceph/qa/tasks/mon_clock_skew_check.py b/ceph/qa/tasks/mon_clock_skew_check.py index f7862cb13..59d4169d1 100644 --- a/ceph/qa/tasks/mon_clock_skew_check.py +++ b/ceph/qa/tasks/mon_clock_skew_check.py @@ -2,8 +2,8 @@ Handle clock skews in monitors. """ import logging -import ceph_manager import time +from tasks import ceph_manager from teuthology import misc as teuthology log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/mon_recovery.py b/ceph/qa/tasks/mon_recovery.py index e09e9877b..fa7aa1a8d 100644 --- a/ceph/qa/tasks/mon_recovery.py +++ b/ceph/qa/tasks/mon_recovery.py @@ -2,7 +2,7 @@ Monitor recovery """ import logging -import ceph_manager +from tasks import ceph_manager from teuthology import misc as teuthology @@ -55,7 +55,7 @@ def task(ctx, config): manager.kill_mon(m) log.info('forming a minimal quorum for %s, then adding monitors' % mons) - qnum = (len(mons) / 2) + 1 + qnum = (len(mons) // 2) + 1 num = 0 for m in mons: manager.revive_mon(m) diff --git a/ceph/qa/tasks/multibench.py b/ceph/qa/tasks/multibench.py index 53b1aa521..c2a7299f1 100644 --- a/ceph/qa/tasks/multibench.py +++ b/ceph/qa/tasks/multibench.py @@ -3,11 +3,12 @@ Multibench testing """ import contextlib import logging -import radosbench import time import copy import gevent +from tasks import radosbench + log = logging.getLogger(__name__) @contextlib.contextmanager diff --git a/ceph/qa/tasks/object_source_down.py b/ceph/qa/tasks/object_source_down.py index 82ce43263..e4519bb6f 100644 --- a/ceph/qa/tasks/object_source_down.py +++ b/ceph/qa/tasks/object_source_down.py @@ -2,10 +2,10 @@ Test Object locations going down """ import logging -import ceph_manager import time from teuthology import misc as teuthology -from util.rados import rados +from tasks import ceph_manager +from tasks.util.rados import rados log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/openssl_keys.py b/ceph/qa/tasks/openssl_keys.py index 657aa7d55..3cc4ed8a5 100644 --- a/ceph/qa/tasks/openssl_keys.py +++ b/ceph/qa/tasks/openssl_keys.py @@ -64,8 +64,9 @@ class OpenSSLKeys(Task): # use testdir/ca as a working directory self.cadir = '/'.join((misc.get_testdir(self.ctx), 'ca')) - - for name, config in self.config.items(): + # make sure self-signed certs get added first, they don't have 'ca' field + configs = sorted(self.config.items(), key=lambda x: 'ca' in x[1]) + for name, config in configs: # names must be unique to avoid clobbering each others files if name in self.ctx.ssl_certificates: raise ConfigError('ssl: duplicate certificate name {}'.format(name)) diff --git a/ceph/qa/tasks/osd_backfill.py b/ceph/qa/tasks/osd_backfill.py index 5ad5b7998..b33e1c912 100644 --- a/ceph/qa/tasks/osd_backfill.py +++ b/ceph/qa/tasks/osd_backfill.py @@ -2,8 +2,8 @@ Osd backfill test """ import logging -import ceph_manager import time +from tasks import ceph_manager from teuthology import misc as teuthology diff --git a/ceph/qa/tasks/osd_failsafe_enospc.py b/ceph/qa/tasks/osd_failsafe_enospc.py index 6aec322ea..4b2cdb983 100644 --- a/ceph/qa/tasks/osd_failsafe_enospc.py +++ b/ceph/qa/tasks/osd_failsafe_enospc.py @@ -7,7 +7,7 @@ import six import time from teuthology.orchestra import run -from util.rados import rados +from tasks.util.rados import rados from teuthology import misc as teuthology log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/osd_recovery.py b/ceph/qa/tasks/osd_recovery.py index a01fe8fea..b0623c21b 100644 --- a/ceph/qa/tasks/osd_recovery.py +++ b/ceph/qa/tasks/osd_recovery.py @@ -2,8 +2,8 @@ osd recovery """ import logging -import ceph_manager import time +from tasks import ceph_manager from teuthology import misc as teuthology diff --git a/ceph/qa/tasks/peer.py b/ceph/qa/tasks/peer.py index e5344a1ce..6b19096b1 100644 --- a/ceph/qa/tasks/peer.py +++ b/ceph/qa/tasks/peer.py @@ -5,9 +5,9 @@ import logging import json import time -import ceph_manager +from tasks import ceph_manager +from tasks.util.rados import rados from teuthology import misc as teuthology -from util.rados import rados log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/peering_speed_test.py b/ceph/qa/tasks/peering_speed_test.py index ab5323856..9dc658361 100644 --- a/ceph/qa/tasks/peering_speed_test.py +++ b/ceph/qa/tasks/peering_speed_test.py @@ -6,7 +6,7 @@ import time log = logging.getLogger(__name__) -from args import argify +from teuthology.task.args import argify POOLNAME = "POOLNAME" ARGS = [ diff --git a/ceph/qa/tasks/qemu.py b/ceph/qa/tasks/qemu.py index 0b77ff5d3..06c79cb5b 100644 --- a/ceph/qa/tasks/qemu.py +++ b/ceph/qa/tasks/qemu.py @@ -1,7 +1,6 @@ """ Qemu task """ -from cStringIO import StringIO import contextlib import logging @@ -9,13 +8,12 @@ import os import yaml import time -from teuthology import misc as teuthology -from teuthology import contextutil from tasks import rbd -from teuthology.orchestra import run +from tasks.util.workunit import get_refspec_after_overrides +from teuthology import contextutil +from teuthology import misc as teuthology from teuthology.config import config as teuth_config - -from util.workunit import get_refspec_after_overrides +from teuthology.orchestra import run log = logging.getLogger(__name__) @@ -122,7 +120,7 @@ def generate_iso(ctx, config): userdata_path = os.path.join(testdir, 'qemu', 'userdata.' + client) metadata_path = os.path.join(testdir, 'qemu', 'metadata.' + client) - with open(os.path.join(src_dir, 'userdata_setup.yaml'), 'rb') as f: + with open(os.path.join(src_dir, 'userdata_setup.yaml')) as f: test_setup = ''.join(f.readlines()) # configuring the commands to setup the nfs mount mnt_dir = "/export/{client}".format(client=client) @@ -130,7 +128,7 @@ def generate_iso(ctx, config): mnt_dir=mnt_dir ) - with open(os.path.join(src_dir, 'userdata_teardown.yaml'), 'rb') as f: + with open(os.path.join(src_dir, 'userdata_teardown.yaml')) as f: test_teardown = ''.join(f.readlines()) user_data = test_setup @@ -172,7 +170,7 @@ def generate_iso(ctx, config): user_data = user_data.format( ceph_branch=ctx.config.get('branch'), ceph_sha1=ctx.config.get('sha1')) - teuthology.write_file(remote, userdata_path, StringIO(user_data)) + teuthology.write_file(remote, userdata_path, user_data) with open(os.path.join(src_dir, 'metadata.yaml'), 'rb') as f: teuthology.write_file(remote, metadata_path, f) diff --git a/ceph/qa/tasks/rados.py b/ceph/qa/tasks/rados.py index 153f0cfa2..edce8dc80 100644 --- a/ceph/qa/tasks/rados.py +++ b/ceph/qa/tasks/rados.py @@ -164,8 +164,8 @@ def task(ctx, config): '--objects', str(config.get('objects', 500)), '--max-in-flight', str(config.get('max_in_flight', 16)), '--size', str(object_size), - '--min-stride-size', str(config.get('min_stride_size', object_size / 10)), - '--max-stride-size', str(config.get('max_stride_size', object_size / 5)), + '--min-stride-size', str(config.get('min_stride_size', object_size // 10)), + '--max-stride-size', str(config.get('max_stride_size', object_size // 5)), '--max-seconds', str(config.get('max_seconds', 0)) ]) @@ -201,11 +201,11 @@ def task(ctx, config): if config.get('write_append_excl', True): if 'write' in weights: - weights['write'] = weights['write'] / 2 + weights['write'] = weights['write'] // 2 weights['write_excl'] = weights['write'] if 'append' in weights: - weights['append'] = weights['append'] / 2 + weights['append'] = weights['append'] // 2 weights['append_excl'] = weights['append'] for op, weight in weights.items(): diff --git a/ceph/qa/tasks/radosbench.py b/ceph/qa/tasks/radosbench.py index 90b21a5d8..a2c4f0693 100644 --- a/ceph/qa/tasks/radosbench.py +++ b/ceph/qa/tasks/radosbench.py @@ -83,15 +83,13 @@ def task(ctx, config): else: pool = manager.create_pool_with_unique_name(erasure_code_profile_name=profile_name) - size = config.get('size', 65536) concurrency = config.get('concurrency', 16) osize = config.get('objectsize', 65536) - sizeargs = ['-b', str(size)] - if osize != 0 and osize != size: - # only use -O if this varies from size. kludgey workaround the - # fact that -O was -o in older releases. - sizeargs.extend(['-O', str(osize)]) - + if osize == 0: + objectsize = [] + else: + objectsize = ['--object-size', str(osize)] + size = ['-b', str(config.get('size', 65536))] # If doing a reading run then populate data if runtype != "write": proc = remote.run( @@ -102,9 +100,9 @@ def task(ctx, config): '{tdir}/archive/coverage', 'rados', '--no-log-to-stderr', - '--name', role] - + sizeargs + - ['-t', str(concurrency)] + + '--name', role] + + ['-t', str(concurrency)] + + size + objectsize + ['-p' , pool, 'bench', str(60), "write", "--no-cleanup" ]).format(tdir=testdir), @@ -112,7 +110,8 @@ def task(ctx, config): logger=log.getChild('radosbench.{id}'.format(id=id_)), wait=True ) - sizeargs = [] + size = [] + objectsize = [] proc = remote.run( args=[ @@ -123,7 +122,7 @@ def task(ctx, config): 'rados', '--no-log-to-stderr', '--name', role] - + sizeargs + + + size + objectsize + ['-p' , pool, 'bench', str(config.get('time', 360)), runtype, ] + write_to_omap + cleanup).format(tdir=testdir), diff --git a/ceph/qa/tasks/radosgw_admin.py b/ceph/qa/tasks/radosgw_admin.py index e91c3d835..092d83fd8 100644 --- a/ceph/qa/tasks/radosgw_admin.py +++ b/ceph/qa/tasks/radosgw_admin.py @@ -14,11 +14,12 @@ import json import logging import time import datetime -import Queue +from six.moves import queue import sys +import six -from cStringIO import StringIO +from io import BytesIO import boto.exception import boto.s3.connection @@ -27,7 +28,7 @@ import boto.s3.acl import httplib2 -from util.rgw import rgwadmin, get_user_summary, get_user_successful_ops +from tasks.util.rgw import rgwadmin, get_user_summary, get_user_successful_ops log = logging.getLogger(__name__) @@ -97,7 +98,7 @@ class usage_acc: def update(self, c, cat, user, out, b_in, err): x = self.c2x(c, cat) usage_acc_update2(x, out, b_in, err) - if not err and cat == 'create_bucket' and not x.has_key('owner'): + if not err and cat == 'create_bucket' and 'owner' not in x: x['owner'] = user def make_entry(self, cat, bucket, user, out, b_in, err): if cat == 'create_bucket' and err: @@ -115,7 +116,7 @@ class usage_acc: def get_usage(self): return self.results def compare_results(self, results): - if not results.has_key('entries') or not results.has_key('summary'): + if 'entries' not in results or 'summary' not in results: return ['Missing entries or summary'] r = [] for e in self.results['entries']: @@ -195,7 +196,7 @@ def ignore_this_entry(cat, bucket, user, out, b_in, err): pass class requestlog_queue(): def __init__(self, add): - self.q = Queue.Queue(1000) + self.q = queue.Queue(1000) self.adder = add def handle_request_data(self, request, response, error=False): now = datetime.datetime.now() @@ -214,8 +215,9 @@ class requestlog_queue(): if 'Content-Length' in j['o'].headers: bytes_out = int(j['o'].headers['Content-Length']) bytes_in = 0 - if 'content-length' in j['i'].msg.dict: - bytes_in = int(j['i'].msg.dict['content-length']) + msg = j['i'].msg if six.PY3 else j['i'].msg.dict + if 'content-length'in msg: + bytes_in = int(msg['content-length']) log.info('RL: %s %s %s bytes_out=%d bytes_in=%d failed=%r' % (cat, bucket, user, bytes_out, bytes_in, j['e'])) if add_entry == None: @@ -243,7 +245,7 @@ def get_acl(key): Helper function to get the xml acl from a key, ensuring that the xml version tag is removed from the acl response """ - raw_acl = key.get_xml_acl() + raw_acl = six.ensure_str(key.get_xml_acl()) def remove_version(string): return string.split( @@ -271,7 +273,7 @@ def task(ctx, config): clients_from_config = config.keys() # choose first client as default - client = clients_from_config[0] + client = next(iter(clients_from_config)) # once the client is chosen, pull the host name and assigned port out of # the role_endpoints that were assigned by the rgw task @@ -662,7 +664,7 @@ def task(ctx, config): rl.log_and_clear("put_obj", bucket_name, user1) # fetch it too (for usage stats presently) - s = key.get_contents_as_string() + s = key.get_contents_as_string(encoding='ascii') rl.log_and_clear("get_obj", bucket_name, user1) assert s == object_name # list bucket too (for usage stats presently) @@ -798,7 +800,7 @@ def task(ctx, config): rl.log_and_clear("put_acls", bucket_name, user1) (err, out) = rgwadmin(ctx, client, - ['policy', '--bucket', bucket.name, '--object', key.key], + ['policy', '--bucket', bucket.name, '--object', six.ensure_str(key.key)], check_status=True, format='xml') acl = get_acl(key) @@ -811,7 +813,7 @@ def task(ctx, config): rl.log_and_clear("put_acls", bucket_name, user1) (err, out) = rgwadmin(ctx, client, - ['policy', '--bucket', bucket.name, '--object', key.key], + ['policy', '--bucket', bucket.name, '--object', six.ensure_str(key.key)], check_status=True, format='xml') acl = get_acl(key) @@ -941,7 +943,7 @@ def task(ctx, config): # get bucket and object to test if user keys are preserved bucket = connection3.get_bucket(bucket_name + '6') - s = key1.get_contents_as_string() + s = key1.get_contents_as_string(encoding='ascii') rl.log_and_clear("get_obj", bucket_name + '6', user4) assert s == object_name1 @@ -973,12 +975,12 @@ def task(ctx, config): # test if user 2 and user4 can still access their bucket and objects after rename fails bucket = connection3.get_bucket(bucket_name + '6') - s = key1.get_contents_as_string() + s = key1.get_contents_as_string(encoding='ascii') rl.log_and_clear("get_obj", bucket_name + '6', user4) assert s == object_name1 bucket = connection2.get_bucket(bucket_name + '7') - s = key2.get_contents_as_string() + s = key2.get_contents_as_string(encoding='ascii') rl.log_and_clear("get_obj", bucket_name + '7', user2) assert s == object_name2 @@ -1038,7 +1040,7 @@ def task(ctx, config): out['placement_pools'].append(rule) (err, out) = rgwadmin(ctx, client, ['zone', 'set'], - stdin=StringIO(json.dumps(out)), + stdin=BytesIO(six.ensure_binary(json.dumps(out))), check_status=True) (err, out) = rgwadmin(ctx, client, ['zone', 'get']) @@ -1071,16 +1073,15 @@ def main(): client0 = remote.Remote(user + host) ctx = config ctx.cluster=cluster.Cluster(remotes=[(client0, - [ 'ceph.client.rgw.%s' % (host), ]),]) - + [ 'ceph.client.rgw.%s' % (host), ]),]) ctx.rgw = argparse.Namespace() endpoints = {} endpoints['ceph.client.rgw.%s' % host] = (host, 80) ctx.rgw.role_endpoints = endpoints ctx.rgw.realm = None ctx.rgw.regions = {'region0': { 'api name': 'api1', - 'is master': True, 'master zone': 'r0z0', - 'zones': ['r0z0', 'r0z1'] }} + 'is master': True, 'master zone': 'r0z0', + 'zones': ['r0z0', 'r0z1'] }} ctx.rgw.config = {'ceph.client.rgw.%s' % host: {'system user': {'name': '%s-system-user' % host}}} task(config, None) exit() diff --git a/ceph/qa/tasks/radosgw_admin_rest.py b/ceph/qa/tasks/radosgw_admin_rest.py index 50f88ea85..df3fa7a1d 100644 --- a/ceph/qa/tasks/radosgw_admin_rest.py +++ b/ceph/qa/tasks/radosgw_admin_rest.py @@ -19,7 +19,7 @@ import time from boto.connection import AWSAuthConnection from teuthology import misc as teuthology -from util.rgw import get_user_summary, get_user_successful_ops, rgwadmin +from tasks.util.rgw import get_user_summary, get_user_successful_ops, rgwadmin log = logging.getLogger(__name__) @@ -130,7 +130,7 @@ def task(ctx, config): clients = config.keys() # just use the first client... - client = clients[0] + client = next(iter(clients)) ## admin_user = 'ada' diff --git a/ceph/qa/tasks/ragweed.py b/ceph/qa/tasks/ragweed.py index cc63ce742..052992a49 100644 --- a/ceph/qa/tasks/ragweed.py +++ b/ceph/qa/tasks/ragweed.py @@ -1,13 +1,14 @@ """ Run a set of s3 tests on rgw. """ -from cStringIO import StringIO +from io import BytesIO from configobj import ConfigObj import base64 import contextlib import logging import os import random +import six import string from teuthology import misc as teuthology @@ -100,8 +101,8 @@ def _config_user(ragweed_conf, section, user): ragweed_conf[section].setdefault('user_id', user) ragweed_conf[section].setdefault('email', '{user}+test@test.test'.format(user=user)) ragweed_conf[section].setdefault('display_name', 'Mr. {user}'.format(user=user)) - ragweed_conf[section].setdefault('access_key', ''.join(random.choice(string.uppercase) for i in range(20))) - ragweed_conf[section].setdefault('secret_key', base64.b64encode(os.urandom(40))) + ragweed_conf[section].setdefault('access_key', ''.join(random.choice(string.ascii_uppercase) for i in range(20))) + ragweed_conf[section].setdefault('secret_key', base64.b64encode(os.urandom(40)).decode('ascii')) @contextlib.contextmanager @@ -112,7 +113,7 @@ def create_users(ctx, config, run_stages): assert isinstance(config, dict) for client, properties in config['config'].items(): - run_stages[client] = string.split(properties.get('stages', 'prepare,check'), ',') + run_stages[client] = properties.get('stages', 'prepare,check').split(',') log.info('Creating rgw users...') testdir = teuthology.get_testdir(ctx) @@ -155,7 +156,7 @@ def create_users(ctx, config, run_stages): if not 'check' in run_stages[client]: # only remove user if went through the check stage continue - for user in users.itervalues(): + for user in users.values(): uid = '{user}.{client}'.format(user=user, client=client) ctx.cluster.only(client).run( args=[ @@ -200,7 +201,7 @@ def configure(ctx, config, run_stages): if properties is not None and 'slow_backend' in properties: ragweed_conf['fixtures']['slow backend'] = properties['slow_backend'] - conf_fp = StringIO() + conf_fp = BytesIO() ragweed_conf.write(conf_fp) teuthology.write_file( remote=remote, @@ -211,7 +212,7 @@ def configure(ctx, config, run_stages): log.info('Configuring boto...') boto_src = os.path.join(os.path.dirname(__file__), 'boto.cfg.template') for client, properties in config['clients'].items(): - with open(boto_src, 'rb') as f: + with open(boto_src, 'r') as f: (remote,) = ctx.cluster.only(client).remotes.keys() conf = f.read().format( idle_timeout=config.get('idle_timeout', 30) @@ -248,7 +249,7 @@ def run_tests(ctx, config, run_stages): testdir = teuthology.get_testdir(ctx) attrs = ["!fails_on_rgw"] for client, client_config in config.items(): - stages = string.join(run_stages[client], ',') + stages = ','.join(run_stages[client]) args = [ 'RAGWEED_CONF={tdir}/archive/ragweed.{client}.conf'.format(tdir=testdir, client=client), 'RAGWEED_STAGES={stages}'.format(stages=stages), diff --git a/ceph/qa/tasks/rbd.py b/ceph/qa/tasks/rbd.py index faa094dab..6d066a6f2 100644 --- a/ceph/qa/tasks/rbd.py +++ b/ceph/qa/tasks/rbd.py @@ -7,7 +7,7 @@ import os import tempfile import sys -from cStringIO import StringIO +from io import BytesIO from teuthology.orchestra import run from teuthology import misc as teuthology from teuthology import contextutil @@ -303,12 +303,12 @@ def canonical_path(ctx, role, path): representing the given role. A canonical path contains no . or .. components, and includes no symbolic links. """ - version_fp = StringIO() + version_fp = BytesIO() ctx.cluster.only(role).run( args=[ 'readlink', '-f', path ], stdout=version_fp, ) - canonical_path = version_fp.getvalue().rstrip('\n') + canonical_path = six.ensure_str(version_fp.getvalue()).rstrip('\n') version_fp.close() return canonical_path @@ -413,9 +413,10 @@ def run_xfstests_one_client(ctx, role, properties): log.info(' randomize: {randomize}'.format(randomize=randomize)) if exclude_list: - with tempfile.NamedTemporaryFile(bufsize=0, prefix='exclude') as exclude_file: + with tempfile.NamedTemporaryFile(mode='w', prefix='exclude') as exclude_file: for test in exclude_list: exclude_file.write("{}\n".format(test)) + exclude_file.flush() remote.put_file(exclude_file.name, exclude_file.name) # Note that the device paths are interpreted using diff --git a/ceph/qa/tasks/rbd_fio.py b/ceph/qa/tasks/rbd_fio.py index 791cfd00a..decc60c26 100644 --- a/ceph/qa/tasks/rbd_fio.py +++ b/ceph/qa/tasks/rbd_fio.py @@ -9,7 +9,6 @@ import contextlib import json import logging import os -import StringIO from teuthology.parallel import parallel from teuthology import misc as teuthology @@ -77,10 +76,8 @@ def get_ioengine_package_name(ioengine, remote): def run_rbd_map(remote, image, iodepth): iodepth = max(iodepth, 128) # RBD_QUEUE_DEPTH_DEFAULT - out = StringIO.StringIO() - remote.run(args=['sudo', 'rbd', 'device', 'map', '-o', - 'queue_depth={}'.format(iodepth), image], stdout=out) - dev = out.getvalue().rstrip('\n') + dev = remote.sh(['sudo', 'rbd', 'device', 'map', '-o', + 'queue_depth={}'.format(iodepth), image]).rstrip('\n') teuthology.sudo_write_file( remote, '/sys/block/{}/queue/nr_requests'.format(os.path.basename(dev)), @@ -94,7 +91,7 @@ def run_fio(remote, config, rbd_test_dir): get the fio from github, generate binary, and use it to run on the generated fio config file """ - fio_config=NamedTemporaryFile(prefix='fio_rbd_', dir='/tmp/', delete=False) + fio_config=NamedTemporaryFile(mode='w', prefix='fio_rbd_', dir='/tmp/', delete=False) fio_config.write('[global]\n') if config.get('io-engine'): ioengine=config['io-engine'] @@ -214,9 +211,8 @@ def run_fio(remote, config, rbd_test_dir): remote.run(args=['sudo', run.Raw('{tdir}/fio-fio-{v}/fio {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))]) remote.run(args=['ceph', '-s']) finally: - out=StringIO.StringIO() - remote.run(args=['rbd', 'device', 'list', '--format=json'], stdout=out) - mapped_images = json.loads(out.getvalue()) + out = remote.sh('rbd device list --format=json') + mapped_images = json.loads(out) if mapped_images: log.info("Unmapping rbd images on {sn}".format(sn=sn)) for image in mapped_images: diff --git a/ceph/qa/tasks/rbd_mirror.py b/ceph/qa/tasks/rbd_mirror.py index 880ec3058..5d6d1b2b6 100644 --- a/ceph/qa/tasks/rbd_mirror.py +++ b/ceph/qa/tasks/rbd_mirror.py @@ -8,7 +8,7 @@ from teuthology.orchestra import run from teuthology import misc from teuthology.exceptions import ConfigError from teuthology.task import Task -from util import get_remote_for_role +from tasks.util import get_remote_for_role log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/rebuild_mondb.py b/ceph/qa/tasks/rebuild_mondb.py index 7877f22ab..008e312e2 100644 --- a/ceph/qa/tasks/rebuild_mondb.py +++ b/ceph/qa/tasks/rebuild_mondb.py @@ -8,7 +8,7 @@ import os.path import shutil import tempfile -import ceph_manager +from tasks import ceph_manager from teuthology import misc as teuthology log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/reg11184.py b/ceph/qa/tasks/reg11184.py index 1059fda71..86cfbf39a 100644 --- a/ceph/qa/tasks/reg11184.py +++ b/ceph/qa/tasks/reg11184.py @@ -12,7 +12,7 @@ import time from teuthology.exceptions import CommandFailedError from teuthology.orchestra import run from teuthology import misc as teuthology -from util.rados import rados +from tasks.util.rados import rados import os diff --git a/ceph/qa/tasks/rep_lost_unfound_delete.py b/ceph/qa/tasks/rep_lost_unfound_delete.py index 8ed55145b..d422a33bb 100644 --- a/ceph/qa/tasks/rep_lost_unfound_delete.py +++ b/ceph/qa/tasks/rep_lost_unfound_delete.py @@ -2,11 +2,12 @@ Lost_unfound """ import logging -from teuthology.orchestra import run -import ceph_manager import time + +from tasks import ceph_manager +from tasks.util.rados import rados from teuthology import misc as teuthology -from util.rados import rados +from teuthology.orchestra import run log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/resolve_stuck_peering.py b/ceph/qa/tasks/resolve_stuck_peering.py index 9b383fd68..d140544c4 100644 --- a/ceph/qa/tasks/resolve_stuck_peering.py +++ b/ceph/qa/tasks/resolve_stuck_peering.py @@ -5,7 +5,7 @@ import logging import time from teuthology import misc as teuthology -from util.rados import rados +from tasks.util.rados import rados log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/rgw.py b/ceph/qa/tasks/rgw.py index 2d99d9962..75004c87a 100644 --- a/ceph/qa/tasks/rgw.py +++ b/ceph/qa/tasks/rgw.py @@ -9,11 +9,11 @@ from teuthology.orchestra import run from teuthology import misc as teuthology from teuthology import contextutil from teuthology.exceptions import ConfigError -from util import get_remote_for_role -from util.rgw import rgwadmin, wait_for_radosgw -from util.rados import (create_ec_pool, - create_replicated_pool, - create_cache_pool) +from tasks.util import get_remote_for_role +from tasks.util.rgw import rgwadmin, wait_for_radosgw +from tasks.util.rados import (create_ec_pool, + create_replicated_pool, + create_cache_pool) log = logging.getLogger(__name__) @@ -104,9 +104,9 @@ def start_rgw(ctx, config, clients): ]) - if client_config.get('dns-name'): + if client_config.get('dns-name') is not None: rgw_cmd.extend(['--rgw-dns-name', endpoint.dns_name]) - if client_config.get('dns-s3website-name'): + if client_config.get('dns-s3website-name') is not None: rgw_cmd.extend(['--rgw-dns-s3website-name', endpoint.website_dns_name]) @@ -223,9 +223,8 @@ def assign_endpoints(ctx, config, default_cert): dns_name += remote.hostname website_dns_name = client_config.get('dns-s3website-name') - if website_dns_name: - if len(website_dns_name) == 0 or website_dns_name.endswith('.'): - website_dns_name += remote.hostname + if website_dns_name is not None and (len(website_dns_name) == 0 or website_dns_name.endswith('.')): + website_dns_name += remote.hostname role_endpoints[role] = RGWEndpoint(remote.hostname, port, ssl_certificate, dns_name, website_dns_name) diff --git a/ceph/qa/tasks/rgw_logsocket.py b/ceph/qa/tasks/rgw_logsocket.py index db6bbcb2e..d76e59d7f 100644 --- a/ceph/qa/tasks/rgw_logsocket.py +++ b/ceph/qa/tasks/rgw_logsocket.py @@ -1,11 +1,11 @@ """ rgw s3tests logging wrappers """ -from cStringIO import StringIO +from io import BytesIO from configobj import ConfigObj import contextlib import logging -import s3tests +from tasks import s3tests from teuthology import misc as teuthology from teuthology import contextutil @@ -68,7 +68,7 @@ def run_tests(ctx, config): s3tests.run_tests(ctx, config) - netcat_out = StringIO() + netcat_out = BytesIO() for client, client_config in config.items(): ctx.cluster.only(client).run( diff --git a/ceph/qa/tasks/rgw_multisite.py b/ceph/qa/tasks/rgw_multisite.py index 5400020db..266d0fb69 100644 --- a/ceph/qa/tasks/rgw_multisite.py +++ b/ceph/qa/tasks/rgw_multisite.py @@ -6,11 +6,11 @@ import logging import random import string from copy import deepcopy -from util.rgw import rgwadmin, wait_for_radosgw -from util.rados import create_ec_pool, create_replicated_pool -from rgw_multi import multisite -from rgw_multi.zone_rados import RadosZone as RadosZone -from rgw_multi.zone_ps import PSZone as PSZone +from tasks.util.rgw import rgwadmin, wait_for_radosgw +from tasks.util.rados import create_ec_pool, create_replicated_pool +from tasks.rgw_multi import multisite +from tasks.rgw_multi.zone_rados import RadosZone as RadosZone +from tasks.rgw_multi.zone_ps import PSZone as PSZone from teuthology.orchestra import run from teuthology import misc diff --git a/ceph/qa/tasks/rgw_multisite_tests.py b/ceph/qa/tasks/rgw_multisite_tests.py index dee6bfaa3..53aedf792 100644 --- a/ceph/qa/tasks/rgw_multisite_tests.py +++ b/ceph/qa/tasks/rgw_multisite_tests.py @@ -9,7 +9,7 @@ from teuthology.exceptions import ConfigError from teuthology.task import Task from teuthology import misc -from rgw_multi import multisite, tests, tests_ps +from tasks.rgw_multi import multisite, tests, tests_ps log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/s3tests.py b/ceph/qa/tasks/s3tests.py index 5adb6fd5b..c0266b6d2 100644 --- a/ceph/qa/tasks/s3tests.py +++ b/ceph/qa/tasks/s3tests.py @@ -1,13 +1,14 @@ """ Run a set of s3 tests on rgw. """ -from cStringIO import StringIO +from io import BytesIO from configobj import ConfigObj import base64 import contextlib import logging import os import random +import six import string from teuthology import misc as teuthology @@ -78,10 +79,14 @@ def _config_user(s3tests_conf, section, user): s3tests_conf[section].setdefault('user_id', user) s3tests_conf[section].setdefault('email', '{user}+test@test.test'.format(user=user)) s3tests_conf[section].setdefault('display_name', 'Mr. {user}'.format(user=user)) - s3tests_conf[section].setdefault('access_key', ''.join(random.choice(string.uppercase) for i in range(20))) - s3tests_conf[section].setdefault('secret_key', base64.b64encode(os.urandom(40))) - s3tests_conf[section].setdefault('totp_serial', ''.join(random.choice(string.digits) for i in range(10))) - s3tests_conf[section].setdefault('totp_seed', base64.b32encode(os.urandom(40))) + s3tests_conf[section].setdefault('access_key', + ''.join(random.choice(string.ascii_uppercase) for i in range(20))) + s3tests_conf[section].setdefault('secret_key', + six.ensure_str(base64.b64encode(os.urandom(40)))) + s3tests_conf[section].setdefault('totp_serial', + ''.join(random.choice(string.digits) for i in range(10))) + s3tests_conf[section].setdefault('totp_seed', + six.ensure_str(base64.b32encode(os.urandom(40)))) s3tests_conf[section].setdefault('totp_seconds', '5') @@ -141,7 +146,7 @@ def create_users(ctx, config): yield finally: for client in config['clients']: - for user in users.itervalues(): + for user in users.values(): uid = '{user}.{client}'.format(user=user, client=client) cluster_name, daemon_type, client_id = teuthology.split_role(client) client_with_id = daemon_type + '.' + client_id @@ -229,7 +234,7 @@ def configure(ctx, config): './bootstrap', ], ) - conf_fp = StringIO() + conf_fp = BytesIO() s3tests_conf.write(conf_fp) teuthology.write_file( remote=remote, @@ -242,13 +247,13 @@ def configure(ctx, config): for client, properties in config['clients'].items(): with open(boto_src, 'rb') as f: (remote,) = ctx.cluster.only(client).remotes.keys() - conf = f.read().format( + conf = six.ensure_str(f.read()).format( idle_timeout=config.get('idle_timeout', 30) ) teuthology.write_file( remote=remote, path='{tdir}/boto.cfg'.format(tdir=testdir), - data=conf, + data=six.ensure_binary(conf), ) try: diff --git a/ceph/qa/tasks/s3tests_java.py b/ceph/qa/tasks/s3tests_java.py index 5ccf46e8a..1f8afd9ee 100644 --- a/ceph/qa/tasks/s3tests_java.py +++ b/ceph/qa/tasks/s3tests_java.py @@ -1,7 +1,7 @@ """ Task for running RGW S3 tests with the AWS Java SDK """ -from cStringIO import StringIO +from io import BytesIO import logging import base64 @@ -117,7 +117,7 @@ class S3tests_java(Task): repo, '{tdir}/s3-tests-java'.format(tdir=testdir), ], - stdout=StringIO() + stdout=BytesIO() ) if client in self.config and self.config[client] is not None: if 'sha1' in self.config[client] and self.config[client]['sha1'] is not None: @@ -156,7 +156,7 @@ class S3tests_java(Task): testdir = teuthology.get_testdir(self.ctx) self.ctx.cluster.only(client).run( args=['{tdir}/s3-tests-java/bootstrap.sh'.format(tdir=testdir)], - stdout=StringIO() + stdout=BytesIO() ) endpoint = self.ctx.rgw.role_endpoints[client] @@ -173,7 +173,7 @@ class S3tests_java(Task): '-file', endpoint.cert.certificate, '-storepass', 'changeit', ], - stdout=StringIO() + stdout=BytesIO() ) def create_users(self): @@ -196,7 +196,7 @@ class S3tests_java(Task): '/home/{local}/s3tests.teuth.config.yaml'.format(local=local_user)) log.debug("S3 Tests Java: s3tests_conf is {s3cfg}".format( s3cfg=s3tests_conf)) - for section, user in self.users.items(): + for section, user in list(self.users.items()): if section in s3tests_conf: s3_user_id = '{user}.{client}'.format( user=user, client=client) @@ -224,7 +224,7 @@ class S3tests_java(Task): log.info('{args}'.format(args=args)) self.ctx.cluster.only(client).run( args=args, - stdout=StringIO() + stdout=BytesIO() ) else: self.users.pop(section) @@ -239,7 +239,7 @@ class S3tests_java(Task): """ access_key = ''.join(random.choice(string.ascii_uppercase) for i in range(20)) - access_secret = base64.b64encode(os.urandom(40)) + access_secret = base64.b64encode(os.urandom(40)).decode('ascii') endpoint = self.ctx.rgw.role_endpoints.get(client) self._set_cfg_entry( @@ -279,8 +279,8 @@ class S3tests_java(Task): with open('s3_tests_tmp.yaml', 'w') as outfile: yaml.dump(cfg_dict, outfile, default_flow_style=False) - conf_fp = StringIO() - with open('s3_tests_tmp.yaml', 'r') as infile: + conf_fp = BytesIO() + with open('s3_tests_tmp.yaml', 'rb') as infile: for line in infile: conf_fp.write(line) @@ -309,7 +309,7 @@ class S3tests_java(Task): '{tdir}/s3-tests-java/config.properties'.format( tdir=testdir) ], - stdout=StringIO() + stdout=BytesIO() ) args = ['cd', '{tdir}/s3-tests-java'.format(tdir=testdir), @@ -346,25 +346,25 @@ class S3tests_java(Task): self.ctx.cluster.only(client).run( args=['radosgw-admin', 'gc', 'process', '--include-all'], - stdout=StringIO() + stdout=BytesIO() ) if gr != 'All': self.ctx.cluster.only(client).run( args=args + ['--tests'] + [gr] + extra_args, - stdout=StringIO() + stdout=BytesIO() ) else: self.ctx.cluster.only(client).run( args=args + extra_args, - stdout=StringIO() + stdout=BytesIO() ) for i in range(2): self.ctx.cluster.only(client).run( args=['radosgw-admin', 'gc', 'process', '--include-all'], - stdout=StringIO() + stdout=BytesIO() ) def remove_tests(self, client): @@ -379,7 +379,7 @@ class S3tests_java(Task): 'cat', self.log_name, run.Raw('&&'), 'rm', self.log_name], - stdout=StringIO() + stdout=BytesIO() ) self.ctx.cluster.only(client).run( @@ -388,7 +388,7 @@ class S3tests_java(Task): '-rf', '{tdir}/s3-tests-java'.format(tdir=testdir), ], - stdout=StringIO() + stdout=BytesIO() ) def delete_users(self, client): @@ -408,7 +408,7 @@ class S3tests_java(Task): '--purge-data', '--cluster', 'ceph', ], - stdout=StringIO() + stdout=BytesIO() ) diff --git a/ceph/qa/tasks/scrub_test.py b/ceph/qa/tasks/scrub_test.py index bc0b3fc09..3d71708ed 100644 --- a/ceph/qa/tasks/scrub_test.py +++ b/ceph/qa/tasks/scrub_test.py @@ -7,7 +7,7 @@ import os import time import tempfile -import ceph_manager +from tasks import ceph_manager from teuthology import misc as teuthology log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/tempest.py b/ceph/qa/tasks/tempest.py index 8550f9b90..a2b55dfb8 100644 --- a/ceph/qa/tasks/tempest.py +++ b/ceph/qa/tasks/tempest.py @@ -4,6 +4,8 @@ Deploy and configure Tempest for Teuthology import contextlib import logging +from six.moves import configparser + from teuthology import misc as teuthology from teuthology import contextutil from teuthology.exceptions import ConfigError @@ -58,15 +60,6 @@ def download(ctx, config): sha1 = cconf.get('sha1') if sha1 is not None: run_in_tempest_dir(ctx, client, [ 'git', 'reset', '--hard', sha1 ]) - - # tox.ini contains a dead link, replace it with the new one - from_url = 'https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt' - to_url = 'https://opendev.org/openstack/requirements/raw/branch/stable/pike/upper-constraints.txt' - run_in_tempest_dir(ctx, client, [ - 'sed', '-i', - run.Raw('"s|{}|{}|"'.format(from_url, to_url)), - 'tox.ini' - ]) try: yield finally: @@ -99,8 +92,12 @@ def setup_logging(ctx, cpar): def to_config(config, params, section, cpar): for (k, v) in config[section].items(): - if (isinstance(v, str)): + if isinstance(v, str): v = v.format(**params) + elif isinstance(v, bool): + v = 'true' if v else 'false' + else: + v = str(v) cpar.set(section, k, v) @contextlib.contextmanager @@ -108,7 +105,6 @@ def configure_instance(ctx, config): assert isinstance(config, dict) log.info('Configuring Tempest') - import ConfigParser for (client, cconfig) in config.items(): run_in_tempest_venv(ctx, client, [ @@ -135,7 +131,7 @@ def configure_instance(ctx, config): 'keystone_public_port': str(public_port), } - cpar = ConfigParser.ConfigParser() + cpar = configparser.ConfigParser() cpar.read(local_conf) setup_logging(ctx, cpar) to_config(cconfig, params, 'auth', cpar) @@ -163,10 +159,8 @@ def run_tempest(ctx, config): get_tempest_dir(ctx) + '/workspace.yaml', '--workspace', 'rgw', - '--regex', - '(tempest.api.object_storage)' + - ''.join([ '(?!{blackitem})'.format(blackitem=blackitem) - for blackitem in blacklist]) + '--regex', '^tempest.api.object_storage', + '--black-regex', '|'.join(blacklist) ]) try: yield @@ -185,13 +179,17 @@ def task(ctx, config): ceph: conf: client: - rgw keystone admin token: ADMIN + rgw keystone api version: 3 rgw keystone accepted roles: admin,Member rgw keystone implicit tenants: true rgw keystone accepted admin roles: admin rgw swift enforce content length: true rgw swift account in url: true rgw swift versioning enabled: true + rgw keystone admin domain: Default + rgw keystone admin user: admin + rgw keystone admin password: ADMIN + rgw keystone admin project: admin tasks: # typically, the task should be preceded with install, ceph, tox, # keystone and rgw. Tox and Keystone are specific requirements diff --git a/ceph/qa/tasks/tests/test_cephadm.py b/ceph/qa/tasks/tests/test_cephadm.py new file mode 100644 index 000000000..403d1915e --- /dev/null +++ b/ceph/qa/tasks/tests/test_cephadm.py @@ -0,0 +1,70 @@ +from tasks import cephadm + +v1 = """ +[registries.search] +registries = ['registry.access.redhat.com', 'registry.redhat.io', 'docker.io', 'quay.io'] + +[registries.insecure] +registries = [] +""" + +v2 = """ +unqualified-search-registries = ["registry.access.redhat.com", "registry.redhat.io", "docker.io", 'quay.io'] + +[[registry]] +prefix = "registry.access.redhat.com" +location = "registry.access.redhat.com" +insecure = false +blocked = false + +[[registry]] +prefix = "registry.redhat.io" +location = "registry.redhat.io" +insecure = false +blocked = false + +[[registry]] +prefix = "docker.io" +location = "docker.io" +insecure = false +blocked = false + +[[registry.mirror]] +location = "vossi04.front.sepia.ceph.com:5000" +insecure = true + +[[registry]] +prefix = "quay.io" +location = "quay.io" +insecure = false +blocked = false +""" + +expected = { + 'unqualified-search-registries': ['registry.access.redhat.com', 'registry.redhat.io', + 'docker.io', 'quay.io'], + 'registry': [ + {'prefix': 'registry.access.redhat.com', + 'location': 'registry.access.redhat.com', + 'insecure': False, + 'blocked': False}, + {'prefix': 'registry.redhat.io', + 'location': 'registry.redhat.io', + 'insecure': False, + 'blocked': False}, + {'prefix': 'docker.io', + 'location': 'docker.io', + 'insecure': False, + 'blocked': False, + 'mirror': [{'location': 'vossi04.front.sepia.ceph.com:5000', + 'insecure': True}]}, + {'prefix': 'quay.io', + 'location': 'quay.io', + 'insecure': False, + 'blocked': False} + ] +} + +def test_add_mirror(): + assert cephadm.registries_add_mirror_to_docker_io(v1, 'vossi04.front.sepia.ceph.com:5000') == expected + assert cephadm.registries_add_mirror_to_docker_io(v2, 'vossi04.front.sepia.ceph.com:5000') == expected diff --git a/ceph/qa/tasks/tests/test_devstack.py b/ceph/qa/tasks/tests/test_devstack.py index 117b30768..39b94a64c 100644 --- a/ceph/qa/tasks/tests/test_devstack.py +++ b/ceph/qa/tasks/tests/test_devstack.py @@ -1,6 +1,6 @@ from textwrap import dedent -from .. import devstack +from tasks import devstack class TestDevstack(object): diff --git a/ceph/qa/tasks/tests/test_radosgw_admin.py b/ceph/qa/tasks/tests/test_radosgw_admin.py index 59f357891..2ed0ebd52 100644 --- a/ceph/qa/tasks/tests/test_radosgw_admin.py +++ b/ceph/qa/tasks/tests/test_radosgw_admin.py @@ -1,6 +1,10 @@ -from mock import Mock +import six +if six.PY3: + from unittest.mock import Mock +else: + from mock import Mock -from .. import radosgw_admin +from tasks import radosgw_admin acl_with_version = """fooFoofooFooFULL_CONTROL """ # noqa diff --git a/ceph/qa/tasks/thrashosds.py b/ceph/qa/tasks/thrashosds.py index 221b02513..aa7ec437a 100644 --- a/ceph/qa/tasks/thrashosds.py +++ b/ceph/qa/tasks/thrashosds.py @@ -176,7 +176,7 @@ def task(ctx, config): for remote in ctx.cluster.remotes.keys(): log.debug('checking console status of %s' % remote.shortname) if not remote.console.check_status(): - log.warn('Failed to get console status for %s', + log.warning('Failed to get console status for %s', remote.shortname) # check that all osd remotes have a valid console diff --git a/ceph/qa/tasks/tox.py b/ceph/qa/tasks/tox.py index 81d712f44..36c226d0b 100644 --- a/ceph/qa/tasks/tox.py +++ b/ceph/qa/tasks/tox.py @@ -31,11 +31,11 @@ def task(ctx, config): # yup, we have to deploy tox first. The packaged one, available # on Sepia's Ubuntu machines, is outdated for Keystone/Tempest. tvdir = get_toxvenv_dir(ctx) - ctx.cluster.only(client).run(args=[ 'virtualenv', tvdir ]) + ctx.cluster.only(client).run(args=[ 'virtualenv', '-p', 'python3', tvdir ]) ctx.cluster.only(client).run(args= [ 'source', '{tvdir}/bin/activate'.format(tvdir=tvdir), run.Raw('&&'), - 'pip', 'install', 'tox==2.3.1' + 'pip', 'install', 'tox==3.15.0' ]) # export the path Keystone and Tempest diff --git a/ceph/qa/tasks/util/rgw.py b/ceph/qa/tasks/util/rgw.py index 91652198b..1b287eb6c 100644 --- a/ceph/qa/tasks/util/rgw.py +++ b/ceph/qa/tasks/util/rgw.py @@ -1,8 +1,9 @@ -from cStringIO import StringIO import logging import json import time +from six import StringIO + from teuthology import misc as teuthology log = logging.getLogger(__name__) diff --git a/ceph/qa/tasks/util/test/test_rados.py b/ceph/qa/tasks/util/test/test_rados.py index ee1cfa62a..a8f4cb02d 100644 --- a/ceph/qa/tasks/util/test/test_rados.py +++ b/ceph/qa/tasks/util/test/test_rados.py @@ -26,7 +26,7 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # -from .. import rados +from tasks.util import rados class TestRados(object): diff --git a/ceph/qa/tasks/vault.py b/ceph/qa/tasks/vault.py index 34efc4f97..6af7c074f 100644 --- a/ceph/qa/tasks/vault.py +++ b/ceph/qa/tasks/vault.py @@ -6,10 +6,10 @@ import argparse import contextlib import logging import time -import httplib import json from os import path -from urlparse import urljoin +from six.moves import http_client +from six.moves.urllib.parse import urljoin from teuthology import misc as teuthology from teuthology import contextutil @@ -128,7 +128,7 @@ def setup_vault(ctx, config): """ Mount Transit or KV version 2 secrets engine """ - (cclient, cconfig) = config.items()[0] + (cclient, cconfig) = next(iter(config.items())) engine = cconfig.get('engine') if engine == 'kv': @@ -155,7 +155,7 @@ def setup_vault(ctx, config): def send_req(ctx, cconfig, client, path, body, method='POST'): host, port = ctx.vault.endpoints[client] - req = httplib.HTTPConnection(host, port, timeout=30) + req = http_client.HTTPConnection(host, port, timeout=30) token = cconfig.get('root_token', 'atoken') log.info("Send request to Vault: %s:%s at %s with token: %s", host, port, path, token) headers = {'X-Vault-Token': token} @@ -169,7 +169,7 @@ def send_req(ctx, cconfig, client, path, body, method='POST'): @contextlib.contextmanager def create_secrets(ctx, config): - (cclient, cconfig) = config.items()[0] + (cclient, cconfig) = next(iter(config.items())) engine = cconfig.get('engine') prefix = cconfig.get('prefix') secrets = cconfig.get('secrets') diff --git a/ceph/qa/tasks/vstart_runner.py b/ceph/qa/tasks/vstart_runner.py index df7005e99..a248db9e4 100644 --- a/ceph/qa/tasks/vstart_runner.py +++ b/ceph/qa/tasks/vstart_runner.py @@ -130,7 +130,7 @@ try: from tasks.cephfs.fuse_mount import FuseMount from tasks.cephfs.kernel_mount import KernelMount from tasks.cephfs.filesystem import Filesystem, MDSCluster, CephCluster - from mgr.mgr_test_case import MgrCluster + from tasks.mgr.mgr_test_case import MgrCluster from teuthology.contextutil import MaxWhileTries from teuthology.task import interactive except ImportError: @@ -339,8 +339,8 @@ class LocalRemote(object): shell = any([a for a in args if isinstance(a, Raw)]) # Filter out helper tools that don't exist in a vstart environment - args = [a for a in args if a not in { - 'adjust-ulimits', 'ceph-coverage'}] + args = [a for a in args if a not in ('adjust-ulimits', + 'ceph-coverage')] # Adjust binary path prefix if given a bare program name if "/" not in args[0]: @@ -396,10 +396,22 @@ class LocalRemote(object): return proc - def sh(self, command, log_limit=1024, cwd=None, env=None): + # XXX: for compatibility keep this method same teuthology.orchestra.remote.sh + def sh(self, script, **kwargs): + """ + Shortcut for run method. + + Usage: + my_name = remote.sh('whoami') + remote_date = remote.sh('date') + """ + if 'stdout' not in kwargs: + kwargs['stdout'] = StringIO() + if 'args' not in kwargs: + kwargs['args'] = script + proc = self.run(**kwargs) + return proc.stdout.getvalue() - return misc.sh(command=command, log_limit=log_limit, cwd=cwd, - env=env) class LocalDaemon(object): def __init__(self, daemon_type, daemon_id): @@ -622,7 +634,7 @@ class LocalKernelMount(KernelMount): log.info("I think my launching pid was {0}".format(self.fuse_daemon.subproc.pid)) return path - def mount(self, mount_path=None, mount_fs_name=None, mount_options=[]): + def mount(self, mount_path=None, mount_fs_name=None, mount_options=[], **kwargs): self.setupfs(name=mount_fs_name) log.info('Mounting kclient client.{id} at {remote} {mnt}...'.format( @@ -800,7 +812,7 @@ class LocalFuseMount(FuseMount): check_status=False ) if p.exitstatus != 0: - log.warn("ls conns failed with {0}, assuming none".format(p.exitstatus)) + log.warning("ls conns failed with {0}, assuming none".format(p.exitstatus)) return [] ls_str = six.ensure_str(p.stdout.getvalue().strip()) @@ -866,6 +878,8 @@ class LocalFuseMount(FuseMount): self.gather_mount_info() + self.mounted = True + def _run_python(self, pyscript, py_version='python'): """ Override this to remove the daemon-helper prefix that is used otherwise @@ -1322,7 +1336,7 @@ def exec_test(): for line in lines: if 'ceph-fuse' in line or 'ceph-mds' in line: pid = int(line.split()[0]) - log.warn("Killing stray process {0}".format(line)) + log.warning("Killing stray process {0}".format(line)) os.kill(pid, signal.SIGKILL) # Fire up the Ceph cluster if the user requested it @@ -1386,7 +1400,7 @@ def exec_test(): mounts.append(mount) if os.path.exists(mount.mountpoint): if mount.is_mounted(): - log.warn("unmounting {0}".format(mount.mountpoint)) + log.warning("unmounting {0}".format(mount.mountpoint)) mount.umount_wait() else: os.rmdir(mount.mountpoint) @@ -1452,11 +1466,11 @@ def exec_test(): if hasattr(fn, 'is_for_teuthology') and getattr(fn, 'is_for_teuthology') is True: drop_test = True - log.warn("Dropping test because long running: {method_id}".format(method_id=method.id())) + log.warning("Dropping test because long running: {method_id}".format(method_id=method.id())) if getattr(fn, "needs_trimming", False) is True: drop_test = (os.getuid() != 0) - log.warn("Dropping test because client trim unavailable: {method_id}".format(method_id=method.id())) + log.warning("Dropping test because client trim unavailable: {method_id}".format(method_id=method.id())) if drop_test: # Don't drop the test if it was explicitly requested in arguments diff --git a/ceph/qa/tasks/watch_notify_same_primary.py b/ceph/qa/tasks/watch_notify_same_primary.py index 716097911..7c034961c 100644 --- a/ceph/qa/tasks/watch_notify_same_primary.py +++ b/ceph/qa/tasks/watch_notify_same_primary.py @@ -2,7 +2,7 @@ """ watch_notify_same_primary task """ -from io import BytesIO +from six import StringIO import contextlib import logging @@ -68,8 +68,8 @@ def task(ctx, config): "watch", obj(n)], stdin=run.PIPE, - stdout=BytesIO(), - stderr=BytesIO(), + stdout=StringIO(), + stderr=StringIO(), wait=False) return proc diff --git a/ceph/qa/tasks/watch_notify_stress.py b/ceph/qa/tasks/watch_notify_stress.py index f3cdf0d54..e5e380492 100644 --- a/ceph/qa/tasks/watch_notify_stress.py +++ b/ceph/qa/tasks/watch_notify_stress.py @@ -3,10 +3,10 @@ test_stress_watch task """ import contextlib import logging -import proc_thrasher import six from teuthology.orchestra import run +from teuthology.task import proc_thrasher log = logging.getLogger(__name__) diff --git a/ceph/qa/tox.ini b/ceph/qa/tox.ini index 746cd1b4f..31a1ef4fd 100644 --- a/ceph/qa/tox.ini +++ b/ceph/qa/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = flake8-py2, flake8-py3, mypy +envlist = flake8-py2, flake8-py3, mypy, pytest skipsdist = True [testenv:flake8-py2] @@ -18,3 +18,11 @@ commands=flake8 --select=F,E9 --exclude=venv,.tox basepython = python3 deps = mypy==0.770 commands = mypy {posargs:.} + +[testenv:pytest] +basepython = python2 +deps = + {env:TEUTHOLOGY_GIT:git+https://github.com/ceph/teuthology.git@py2}#egg=teuthology[test] + httplib2 + mock +commands = pytest -vv tasks/tests diff --git a/ceph/qa/valgrind.supp b/ceph/qa/valgrind.supp index 9abb2f238..1e5fc3262 100644 --- a/ceph/qa/valgrind.supp +++ b/ceph/qa/valgrind.supp @@ -557,8 +557,8 @@ Memcheck:Leak match-leak-kinds: reachable ... - fun:*boost*detail*make_external_thread_data* - fun:*boost*detail*add_new_tss_node* + fun:_Znwm + ... fun:*boost*detail*set_tss_data* ... } diff --git a/ceph/qa/workunits/ceph-helpers-root.sh b/ceph/qa/workunits/ceph-helpers-root.sh index 934380e5c..a81f11d47 100755 --- a/ceph/qa/workunits/ceph-helpers-root.sh +++ b/ceph/qa/workunits/ceph-helpers-root.sh @@ -50,15 +50,6 @@ function install_one() { esac } -function install_cmake3_on_xenial { - install_pkg_on_ubuntu \ - ceph-cmake \ - d278b9d28de0f6b88f56dfe1e8bf684a41577210 \ - xenial \ - force \ - cmake -} - function install_pkg_on_ubuntu { local project=$1 shift diff --git a/ceph/qa/workunits/cephadm/test_adoption.sh b/ceph/qa/workunits/cephadm/test_adoption.sh index 83e11a40a..ad147885d 100755 --- a/ceph/qa/workunits/cephadm/test_adoption.sh +++ b/ceph/qa/workunits/cephadm/test_adoption.sh @@ -3,7 +3,7 @@ SCRIPT_NAME=$(basename ${BASH_SOURCE[0]}) SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" CEPHADM_SRC_DIR=${SCRIPT_DIR}/../../../src/cephadm -CORPUS_COMMIT=50c5dd734638939facd1ed32295ce59c9a5986b4 +CORPUS_COMMIT=9cd9ad020d93b0b420924fec55da307aff8bd422 [ -z "$SUDO" ] && SUDO=sudo if [ -z "$CEPHADM" ]; then diff --git a/ceph/qa/workunits/cephadm/test_cephadm.sh b/ceph/qa/workunits/cephadm/test_cephadm.sh index 42f195568..ddffad1da 100755 --- a/ceph/qa/workunits/cephadm/test_cephadm.sh +++ b/ceph/qa/workunits/cephadm/test_cephadm.sh @@ -330,21 +330,21 @@ done ${CEPHADM//--image $IMAGE_MASTER/} deploy \ --name node-exporter.a --fsid $FSID cond="curl 'http://localhost:9100' | grep -q 'Node Exporter'" -is_available "node-exporter" "$cond" 5 +is_available "node-exporter" "$cond" 10 # add prometheus cat ${CEPHADM_SAMPLES_DIR}/prometheus.json | \ ${CEPHADM//--image $IMAGE_MASTER/} deploy \ --name prometheus.a --fsid $FSID --config-json - cond="curl 'localhost:9095/api/v1/query?query=up'" -is_available "prometheus" "$cond" 5 +is_available "prometheus" "$cond" 10 # add grafana cat ${CEPHADM_SAMPLES_DIR}/grafana.json | \ ${CEPHADM//--image $IMAGE_MASTER/} deploy \ --name grafana.a --fsid $FSID --config-json - cond="curl --insecure 'https://localhost:3000' | grep -q 'grafana'" -is_available "grafana" "$cond" 30 +is_available "grafana" "$cond" 50 # add nfs-ganesha nfs_stop @@ -382,6 +382,7 @@ $CEPHADM shell --fsid $FSID -- true $CEPHADM shell --fsid $FSID -- test -d /var/log/ceph expect_false $CEPHADM --timeout 10 shell --fsid $FSID -- sleep 60 $CEPHADM --timeout 60 shell --fsid $FSID -- sleep 10 +$CEPHADM shell --fsid $FSID --mount $TMPDIR -- stat /mnt/$(basename $TMPDIR) ## enter expect_false $CEPHADM enter diff --git a/ceph/qa/workunits/cephtool/test.sh b/ceph/qa/workunits/cephtool/test.sh index 51d8bd7c7..77b36e0ca 100755 --- a/ceph/qa/workunits/cephtool/test.sh +++ b/ceph/qa/workunits/cephtool/test.sh @@ -345,21 +345,20 @@ function test_tiering_1() ceph osd tier add slow cache ceph osd tier add slow cache2 expect_false ceph osd tier add slow2 cache - # test some state transitions - ceph osd tier cache-mode cache writeback - # forward is removed/deprecated + # forward and proxy are removed/deprecated expect_false ceph osd tier cache-mode cache forward expect_false ceph osd tier cache-mode cache forward --yes-i-really-mean-it + expect_false ceph osd tier cache-mode cache proxy + expect_false ceph osd tier cache-mode cache proxy --yes-i-really-mean-it + # test some state transitions + ceph osd tier cache-mode cache writeback expect_false ceph osd tier cache-mode cache readonly - ceph osd tier cache-mode cache proxy + expect_false ceph osd tier cache-mode cache readonly --yes-i-really-mean-it + ceph osd tier cache-mode cache readproxy ceph osd tier cache-mode cache none ceph osd tier cache-mode cache readonly --yes-i-really-mean-it - expect_false ceph osd tier cache-mode cache forward - expect_false ceph osd tier cache-mode cache forward --yes-i-really-mean-it ceph osd tier cache-mode cache none ceph osd tier cache-mode cache writeback - ceph osd tier cache-mode cache proxy - ceph osd tier cache-mode cache writeback expect_false ceph osd tier cache-mode cache none expect_false ceph osd tier cache-mode cache readonly --yes-i-really-mean-it # test with dirty objects in the tier pool @@ -367,7 +366,7 @@ function test_tiering_1() rados -p cache put /etc/passwd /etc/passwd flush_pg_stats # 1 dirty object in pool 'cache' - ceph osd tier cache-mode cache proxy + ceph osd tier cache-mode cache readproxy expect_false ceph osd tier cache-mode cache none expect_false ceph osd tier cache-mode cache readonly --yes-i-really-mean-it ceph osd tier cache-mode cache writeback @@ -376,7 +375,7 @@ function test_tiering_1() rados -p cache cache-flush-evict-all flush_pg_stats # no dirty objects in pool 'cache' - ceph osd tier cache-mode cache proxy + ceph osd tier cache-mode cache readproxy ceph osd tier cache-mode cache none ceph osd tier cache-mode cache readonly --yes-i-really-mean-it TRIES=0 @@ -1108,7 +1107,7 @@ function test_mon_mds() # Removing tier should be permitted because the underlying pool is # replicated (#11504 case) - ceph osd tier cache-mode mds-tier proxy + ceph osd tier cache-mode mds-tier readproxy ceph osd tier remove-overlay fs_metadata ceph osd tier remove fs_metadata mds-tier ceph osd pool delete mds-tier mds-tier --yes-i-really-really-mean-it diff --git a/ceph/qa/workunits/rados/test_envlibrados_for_rocksdb.sh b/ceph/qa/workunits/rados/test_envlibrados_for_rocksdb.sh index 762a44f18..ecbb4098a 100755 --- a/ceph/qa/workunits/rados/test_envlibrados_for_rocksdb.sh +++ b/ceph/qa/workunits/rados/test_envlibrados_for_rocksdb.sh @@ -20,15 +20,7 @@ CURRENT_PATH=`pwd` # for rocksdb case $(distro_id) in ubuntu|debian|devuan) - install git g++ libsnappy-dev zlib1g-dev libbz2-dev libradospp-dev - case $(distro_version) in - *Xenial*) - install_cmake3_on_xenial - ;; - *) - install cmake - ;; - esac + install git g++ libsnappy-dev zlib1g-dev libbz2-dev libradospp-dev cmake ;; centos|fedora|rhel) case $(distro_id) in @@ -68,7 +60,11 @@ echo "Compile rocksdb" if [ -e rocksdb ]; then rm -fr rocksdb fi -git clone https://github.com/facebook/rocksdb.git --depth 1 + +pushd $(dirname /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/bash.sh)/../../../ +git submodule update --init src/rocksdb +popd +git clone $(dirname /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/bash.sh)/../../../src/rocksdb rocksdb # compile code cd rocksdb diff --git a/ceph/qa/workunits/rbd/rbd-nbd.sh b/ceph/qa/workunits/rbd/rbd-nbd.sh index f4fe24811..5475116cd 100755 --- a/ceph/qa/workunits/rbd/rbd-nbd.sh +++ b/ceph/qa/workunits/rbd/rbd-nbd.sh @@ -106,7 +106,7 @@ unmap_device() for s in 0.5 1 2 4 8 16 32; do sleep ${s} - rbd-nbd list-mapped | expect_false grep "${list_dev} $" && return 0 + rbd-nbd list-mapped | expect_false grep "${list_dev} *$" && return 0 done return 1 } diff --git a/ceph/qa/workunits/rgw/test_rgw_orphan_list.sh b/ceph/qa/workunits/rgw/test_rgw_orphan_list.sh new file mode 100755 index 000000000..67750cd0d --- /dev/null +++ b/ceph/qa/workunits/rgw/test_rgw_orphan_list.sh @@ -0,0 +1,505 @@ +#!/usr/bin/env bash + +set -ex + +# if defined, debug messages will be displayed and prepended with the string +# debug="DEBUG" + +huge_size=2222 # in megabytes +big_size=6 # in megabytes + +huge_obj=/tmp/huge_obj.temp.$$ +big_obj=/tmp/big_obj.temp.$$ +empty_obj=/tmp/empty_obj.temp.$$ + +fifo=/tmp/orphan-fifo.$$ +awscli_dir=${HOME}/awscli_temp +export PATH=${PATH}:${awscli_dir} + +rgw_host=$(hostname --fqdn) +rgw_port=80 + +echo "Fully Qualified Domain Name: $rgw_host" + +success() { + echo OK. + exit 0 +} + +######################################################################## +# INSTALL AND CONFIGURE TOOLING + +install_awscli() { + # NB: this does verify authenticity and integrity of downloaded + # file; see + # https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html + here="$(pwd)" + cd "$HOME" + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + mkdir -p $awscli_dir + ./aws/install -i $awscli_dir + cd "$here" +} + +uninstall_awscli() { + here="$(pwd)" + cd "$HOME" + rm -rf $awscli_dir ./aws awscliv2.zip + cd "$here" +} + +sudo dnf install -y s3cmd + +sudo yum install python3-setuptools +sudo yum -y install python3-pip +sudo pip3 install --upgrade setuptools +sudo pip3 install python-swiftclient + +# get ready for transition from s3cmd to awscli +if false ;then + install_awscli + aws --version + uninstall_awscli +fi + +s3config=/tmp/s3config.$$ + +# do not include the port when it is 80; the host base is used in the +# v4 signature and it needs to follow this convention for signatures +# to match +if [ "$rgw_port" -ne 80 ] ;then + s3_host_base="${rgw_host}:${rgw_port}" +else + s3_host_base="$rgw_host" +fi + +cat >${s3config} <$fifo & + set +e # don't allow errors to stop script + while read line ;do + echo "$line" | grep --quiet "part $stop_part " + if [ ${PIPESTATUS[1]} -eq 0 ] ;then + kill -9 $(jobs -p) + break + fi + done <$fifo + set -e + + rm -f $fifo +} + +mys3upload() { + obj=$1 + bucket=$2 + dest_obj=$3 + + mys3cmd put -q $obj s3://${bucket}/$dest_obj +} + +######################################################################## +# PREP + +create_users +dd if=/dev/urandom of=$big_obj bs=1M count=${big_size} +dd if=/dev/urandom of=$huge_obj bs=1M count=${huge_size} +touch $empty_obj + +quick_tests() { + echo TRY A SWIFT COMMAND + myswift upload swift-plain-ctr $big_obj --object-name swift-obj-test + myswift list + myswift list swift-plain-ctr + + echo TRY A RADOSGW-ADMIN COMMAND + radosgw-admin bucket list # make sure rgw is up and running +} + +######################################################################## +# S3 TESTS + +#################################### +# regular multipart test + +mys3cmd mb s3://multipart-bkt +mys3upload $huge_obj multipart-bkt multipart-obj +mys3cmd ls +mys3cmd ls s3://multipart-bkt + +#################################### +# multipart test with incomplete uploads + +bkt="incomplete-mp-bkt-1" + +mys3cmd mb s3://$bkt +mys3uploadkill $huge_obj $bkt incomplete-mp-obj-1 $fifo 20 +mys3uploadkill $huge_obj $bkt incomplete-mp-obj-2 $fifo 100 + +#################################### +# resharded bucket + +bkt=resharded-bkt-1 + +mys3cmd mb s3://$bkt + +for f in $(seq 8) ; do + dest_obj="reshard-obj-${f}" + mys3cmd put -q $big_obj s3://${bkt}/$dest_obj +done + +radosgw-admin bucket reshard --num-shards 3 --bucket=$bkt --yes-i-really-mean-it +radosgw-admin bucket reshard --num-shards 5 --bucket=$bkt --yes-i-really-mean-it + +#################################### +# versioned bucket + +if true ;then + echo "WARNING: versioned bucket test currently turned off" +else + bkt=versioned-bkt-1 + + mys3cmd mb s3://$bkt + + # bucket-enable-versioning $bkt + + for f in $(seq 3) ;do + for g in $(seq 10) ;do + dest_obj="versioned-obj-${g}" + mys3cmd put -q $big_obj s3://${bkt}/$dest_obj + done + done + + for g in $(seq 1 2 10) ;do + dest_obj="versioned-obj-${g}" + mys3cmd rm s3://${bkt}/$dest_obj + done +fi + +############################################################ +# copy small objects + +o_bkt="orig-bkt-1" +d_bkt="copy-bkt-1" +mys3cmd mb s3://$o_bkt + +for f in $(seq 4) ;do + dest_obj="orig-obj-$f" + mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj +done + +mys3cmd mb s3://$d_bkt + +mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1 +mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3 + +for f in $(seq 5 6) ;do + dest_obj="orig-obj-$f" + mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj +done + +############################################################ +# copy small objects and delete original + +o_bkt="orig-bkt-2" +d_bkt="copy-bkt-2" + +mys3cmd mb s3://$o_bkt + +for f in $(seq 4) ;do + dest_obj="orig-obj-$f" + mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj +done + +mys3cmd mb s3://$d_bkt + +mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1 +mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3 + +for f in $(seq 5 6) ;do + dest_obj="orig-obj-$f" + mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj +done + +mys3cmd rb --recursive s3://${o_bkt} + +############################################################ +# copy multipart objects + +o_bkt="orig-mp-bkt-3" +d_bkt="copy-mp-bkt-3" + +mys3cmd mb s3://$o_bkt + +for f in $(seq 2) ;do + dest_obj="orig-multipart-obj-$f" + mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj +done + +mys3cmd mb s3://$d_bkt + +mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \ + s3://${d_bkt}/copied-multipart-obj-1 + +for f in $(seq 5 5) ;do + dest_obj="orig-multipart-obj-$f" + mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj +done + + +############################################################ +# copy multipart objects and delete original + +o_bkt="orig-mp-bkt-4" +d_bkt="copy-mp-bkt-4" + +mys3cmd mb s3://$o_bkt + +for f in $(seq 2) ;do + dest_obj="orig-multipart-obj-$f" + mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj +done + +mys3cmd mb s3://$d_bkt + +mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \ + s3://${d_bkt}/copied-multipart-obj-1 + +for f in $(seq 5 5) ;do + dest_obj="orig-multipart-obj-$f" + mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj +done + +mys3cmd rb --recursive s3://$o_bkt + +######################################################################## +# SWIFT TESTS + +# 600MB +segment_size=629145600 + +############################################################ +# plain test + +for f in $(seq 4) ;do + myswift upload swift-plain-ctr $big_obj --object-name swift-obj-$f +done + +############################################################ +# zero-len test + +myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/ +myswift upload swift-zerolen-ctr $big_obj --object-name subdir/abc1 +myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/empty1 +myswift upload swift-zerolen-ctr $big_obj --object-name subdir/xyz1 + +############################################################ +# dlo test + +# upload in 300MB segments +myswift upload swift-dlo-ctr $huge_obj --object-name dlo-obj-1 \ + -S $segment_size + +############################################################ +# slo test + +# upload in 300MB segments +myswift upload swift-slo-ctr $huge_obj --object-name slo-obj-1 \ + -S $segment_size --use-slo + +############################################################ +# large object copy test + +# upload in 300MB segments +o_ctr=swift-orig-ctr +o_obj=slo-orig-obj-1 +d_ctr=swift-copy-ctr +d_obj=slo-copy-obj-1 +myswift upload $o_ctr $big_obj --object-name $o_obj + +myswift copy --destination /${d_ctr}/${d_obj} \ + $o_ctr $o_obj + +myswift delete $o_ctr $o_obj + +############################################################ +# huge dlo object copy test + +o_ctr=swift-orig-dlo-ctr-1 +o_obj=dlo-orig-dlo-obj-1 +d_ctr=swift-copy-dlo-ctr-1 +d_obj=dlo-copy-dlo-obj-1 + +myswift upload $o_ctr $huge_obj --object-name $o_obj \ + -S $segment_size + +myswift copy --destination /${d_ctr}/${d_obj} \ + $o_ctr $o_obj + +############################################################ +# huge dlo object copy and orig delete + +o_ctr=swift-orig-dlo-ctr-2 +o_obj=dlo-orig-dlo-obj-2 +d_ctr=swift-copy-dlo-ctr-2 +d_obj=dlo-copy-dlo-obj-2 + +myswift upload $o_ctr $huge_obj --object-name $o_obj \ + -S $segment_size + +myswift copy --destination /${d_ctr}/${d_obj} \ + $o_ctr $o_obj + +myswift delete $o_ctr $o_obj + +############################################################ +# huge slo object copy test + +o_ctr=swift-orig-slo-ctr-1 +o_obj=slo-orig-slo-obj-1 +d_ctr=swift-copy-slo-ctr-1 +d_obj=slo-copy-slo-obj-1 +myswift upload $o_ctr $huge_obj --object-name $o_obj \ + -S $segment_size --use-slo + +myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj + +############################################################ +# huge slo object copy test and orig delete + +o_ctr=swift-orig-slo-ctr-2 +o_obj=slo-orig-slo-obj-2 +d_ctr=swift-copy-slo-ctr-2 +d_obj=slo-copy-slo-obj-2 +myswift upload $o_ctr $huge_obj --object-name $o_obj \ + -S $segment_size --use-slo + +myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj + +myswift delete $o_ctr $o_obj + +######################################################################## +# FORCE GARBAGE COLLECTION + +sleep 6 # since for testing age at which gc can happen is 5 secs +radosgw-admin gc process --include-all + + +######################################## +# DO ORPHAN LIST + +pool="default.rgw.buckets.data" + +rgw-orphan-list $pool + +# we only expect there to be one output file, but loop just in case +ol_error="" +for f in orphan-list-*.out ; do + if [ -s "$f" ] ;then # if file non-empty + ol_error="${ol_error}:$f" + echo "One ore more orphans found in $f:" + cat "$f" + fi +done + +if [ -n "$ol_error" ] ;then + echo "ERROR: orphans found when none expected" + exit 1 +fi + +######################################################################## +# CLEAN UP + +rm -f $empty_obj $big_obj $huge_obj $s3config + +success diff --git a/ceph/selinux/ceph.te b/ceph/selinux/ceph.te index e2a848149..81b4d0067 100644 --- a/ceph/selinux/ceph.te +++ b/ceph/selinux/ceph.te @@ -4,6 +4,7 @@ require { type sysfs_t; type configfs_t; type commplex_main_port_t; + type http_cache_port_t; type rpm_exec_t; type rpm_var_lib_t; type kernel_t; @@ -85,6 +86,7 @@ corenet_tcp_bind_cyphesis_port(ceph_t) corenet_tcp_sendrecv_cyphesis_port(ceph_t) allow ceph_t commplex_main_port_t:tcp_socket name_connect; +allow ceph_t http_cache_port_t:tcp_socket name_connect; corecmd_exec_bin(ceph_t) corecmd_exec_shell(ceph_t) diff --git a/ceph/src/.git_version b/ceph/src/.git_version index d9b1b48b8..ed8078796 100644 --- a/ceph/src/.git_version +++ b/ceph/src/.git_version @@ -1,2 +1,2 @@ -d289bbdec69ed7c1f516e0a093594580a76b78d0 -15.2.3 +7447c15c6ff58d7fce91843b705a268a1917325c +15.2.4 diff --git a/ceph/src/bash_completion/ceph b/ceph/src/bash_completion/ceph index beec700e8..d28c8a4fa 100644 --- a/ceph/src/bash_completion/ceph +++ b/ceph/src/bash_completion/ceph @@ -27,7 +27,7 @@ _ceph() COMPREPLY=( $(compgen -W "${options_noarg} ${options_arg}" -- $cur) ) return 0 fi - declare -A hint_args + declare -a hint_args for (( i=1 ; i #include -#include +#include "include/ceph_fuse.h" #include #define dout_context g_ceph_context diff --git a/ceph/src/ceph_osd.cc b/ceph/src/ceph_osd.cc index 1e9516aa3..afa945dd0 100644 --- a/ceph/src/ceph_osd.cc +++ b/ceph/src/ceph_osd.cc @@ -90,6 +90,8 @@ static void usage() << " --convert-filestore\n" << " run any pending upgrade operations\n" << " --flush-journal flush all data out of journal\n" + << " --osdspec-affinity\n" + << " set affinity to an osdspec\n" << " --dump-journal dump all data of journal\n" << " --mkjournal initialize a new journal\n" << " --check-wants-journal\n" @@ -148,6 +150,7 @@ int main(int argc, const char **argv) bool get_device_fsid = false; string device_path; std::string dump_pg_log; + std::string osdspec_affinity; std::string val; for (std::vector::iterator i = args.begin(); i != args.end(); ) { @@ -155,6 +158,8 @@ int main(int argc, const char **argv) break; } else if (ceph_argparse_flag(args, i, "--mkfs", (char*)NULL)) { mkfs = true; + } else if (ceph_argparse_witharg(args, i, &val, "--osdspec-affinity", (char*)NULL)) { + osdspec_affinity = val; } else if (ceph_argparse_flag(args, i, "--mkjournal", (char*)NULL)) { mkjournal = true; } else if (ceph_argparse_flag(args, i, "--check-allows-journal", (char*)NULL)) { @@ -347,6 +352,7 @@ int main(int argc, const char **argv) derr << "created new key in keyring " << keyring_path << dendl; } } + if (mkfs) { common_init_finish(g_ceph_context); @@ -356,7 +362,7 @@ int main(int argc, const char **argv) } int err = OSD::mkfs(g_ceph_context, store, g_conf().get_val("fsid"), - whoami); + whoami, osdspec_affinity); if (err < 0) { derr << TEXT_RED << " ** ERROR: error creating empty object store in " << data_path << ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl; @@ -442,7 +448,6 @@ flushjournal_out: forker.exit(0); } - if (convertfilestore) { int err = store->mount(); if (err < 0) { diff --git a/ceph/src/cephadm/cephadm b/ceph/src/cephadm/cephadm index 32610e702..e27bc73a6 100755 --- a/ceph/src/cephadm/cephadm +++ b/ceph/src/cephadm/cephadm @@ -87,6 +87,12 @@ cached_stdin = None DATEFMT = '%Y-%m-%dT%H:%M:%S.%f' + +class termcolor: + yellow = '\033[93m' + red = '\033[31m' + end = '\033[0m' + class Error(Exception): pass @@ -231,7 +237,7 @@ class NFSGanesha(object): @staticmethod def get_version(container_id): - # type(str) -> Optional[str] + # type: (str) -> Optional[str] version = None out, err, code = call( [container_path, 'exec', container_id, @@ -243,7 +249,7 @@ class NFSGanesha(object): return version def validate(self): - # type () -> None + # type: () -> None if not is_fsid(self.fsid): raise Error('not an fsid: %s' % self.fsid) if not self.daemon_id: @@ -380,7 +386,7 @@ class CephIscsi(object): @staticmethod def get_version(container_id): - # type(str) -> Optional[str] + # type: (str) -> Optional[str] version = None out, err, code = call( [container_path, 'exec', container_id, @@ -390,7 +396,7 @@ class CephIscsi(object): return version def validate(self): - # type () -> None + # type: () -> None if not is_fsid(self.fsid): raise Error('not an fsid: %s' % self.fsid) if not self.daemon_id: @@ -445,6 +451,7 @@ class CephIscsi(object): @staticmethod def configfs_mount_umount(data_dir, mount=True): + # type: (str, bool) -> List[str] mount_path = os.path.join(data_dir, 'configfs') if mount: cmd = "if ! grep -qs {0} /proc/mounts; then " \ @@ -457,6 +464,7 @@ class CephIscsi(object): ################################## def get_supported_daemons(): + # type: () -> List[str] supported_daemons = list(Ceph.daemons) supported_daemons.extend(Monitoring.components) supported_daemons.append(NFSGanesha.daemon_type) @@ -467,7 +475,7 @@ def get_supported_daemons(): ################################## def attempt_bind(s, address, port): - # type (str) -> None + # type: (socket.socket, str, int) -> None try: s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind((address, port)) @@ -482,9 +490,9 @@ def attempt_bind(s, address, port): s.close() def port_in_use(port_num): - # type (int) -> bool + # type: (int) -> bool """Detect whether a port is in use on the local machine - IPv4 and IPv6""" - logger.info('Verifying port %d ...' % (port_num)) + logger.info('Verifying port %d ...' % port_num) try: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) attempt_bind(s, '0.0.0.0', port_num) @@ -497,6 +505,7 @@ def port_in_use(port_num): return False def check_ip_port(ip, port): + # type: (str, int) -> None if not args.skip_ping_check: logger.info('Verifying IP %s port %d ...' % (ip, port)) if ip.startswith('[') or '::' in ip: @@ -892,6 +901,8 @@ def is_available(what, func): num = 1 while True: if func(): + logger.info('%s is available' + % (what)) break elif num > retry: raise Error('%s not available after %s tries' @@ -928,11 +939,11 @@ def read_config(fn): def pathify(p): # type: (str) -> str - if not p.startswith('/'): - return os.path.join(os.getcwd(), p) - return p + p = os.path.expanduser(p) + return os.path.abspath(p) def get_file_timestamp(fn): + # type: (str) -> Optional[str] try: mt = os.path.getmtime(fn) return datetime.datetime.fromtimestamp( @@ -942,6 +953,7 @@ def get_file_timestamp(fn): return None def try_convert_datetime(s): + # type: (str) -> Optional[str] # This is super irritating because # 1) podman and docker use different formats # 2) python's strptime can't parse either one @@ -1078,16 +1090,44 @@ def infer_fsid(func): return _infer_fsid +def infer_config(func): + """ + If we find a MON daemon, use the config from that container + """ + @wraps(func) + def _infer_config(): + if args.config: + logger.debug('Using specified config: %s' % args.config) + return func() + config = None + if args.fsid: + name = args.name + if not name: + daemon_list = list_daemons(detail=False) + for daemon in daemon_list: + if daemon['name'].startswith('mon.'): + name = daemon['name'] + break + if name: + config = '/var/lib/ceph/{}/{}/config'.format(args.fsid, name) + if config: + logger.info('Inferring config %s' % config) + args.config = config + elif os.path.exists(SHELL_DEFAULT_CONF): + logger.debug('Using default config: %s' % SHELL_DEFAULT_CONF) + args.config = SHELL_DEFAULT_CONF + return func() + + return _infer_config + def _get_default_image(): if DEFAULT_IMAGE_IS_MASTER: - yellow = '\033[93m' - end = '\033[0m' warn = '''This is a development version of cephadm. For information regarding the latest stable release: https://docs.ceph.com/docs/{}/cephadm/install '''.format(LATEST_STABLE_RELEASE) for line in warn.splitlines(): - logger.warning('{}{}{}'.format(yellow, line, end)) + logger.warning('{}{}{}'.format(termcolor.yellow, line, termcolor.end)) return DEFAULT_IMAGE def infer_image(func): @@ -1140,6 +1180,7 @@ def get_last_local_ceph_image(): return None def write_tmp(s, uid, gid): + # type: (str, int, int) -> Any tmp_f = tempfile.NamedTemporaryFile(mode='w', prefix='ceph-tmp') os.fchown(tmp_f.fileno(), uid, gid) @@ -1316,6 +1357,13 @@ def get_unit_name(fsid, daemon_type, daemon_id=None): else: return 'ceph-%s@%s' % (fsid, daemon_type) +def get_unit_name_by_daemon_name(fsid, name): + daemon = get_daemon_description(fsid, name) + try: + return daemon['systemd_unit'] + except KeyError: + raise Error('Failed to get unit name for {}'.format(daemon)) + def check_unit(unit_name): # type: (str) -> Tuple[bool, str, bool] # NOTE: we ignore the exit code here because systemctl outputs @@ -1430,8 +1478,8 @@ def get_daemon_args(fsid, daemon_type, daemon_id): return r def create_daemon_dirs(fsid, daemon_type, daemon_id, uid, gid, - config=None, keyring=None, reconfig=False): - # type: (str, str, Union[int, str], int, int, Optional[str], Optional[str], Optional[bool]) -> None + config=None, keyring=None): + # type: (str, str, Union[int, str], int, int, Optional[str], Optional[str]) -> None data_dir = make_data_dir(fsid, daemon_type, daemon_id, uid=uid, gid=gid) make_log_dir(fsid, uid=uid, gid=gid) @@ -1589,6 +1637,22 @@ def get_container_mounts(fsid, daemon_type, daemon_id, mounts['/run/lvm'] = '/run/lvm' mounts['/run/lock/lvm'] = '/run/lock/lvm' + try: + if args.shared_ceph_folder: # make easy manager modules/ceph-volume development + ceph_folder = pathify(args.shared_ceph_folder) + if os.path.exists(ceph_folder): + mounts[ceph_folder + '/src/ceph-volume/ceph_volume'] = '/usr/lib/python3.6/site-packages/ceph_volume' + mounts[ceph_folder + '/src/pybind/mgr'] = '/usr/share/ceph/mgr' + mounts[ceph_folder + '/src/python-common/ceph'] = '/usr/lib/python3.6/site-packages/ceph' + mounts[ceph_folder + '/monitoring/grafana/dashboards'] = '/etc/grafana/dashboards/ceph-dashboard' + mounts[ceph_folder + '/monitoring/prometheus/alerts'] = '/etc/prometheus/ceph' + else: + logger.error('{}{}{}'.format(termcolor.red, + 'Ceph shared source folder does not exist.', + termcolor.end)) + except AttributeError: + pass + if daemon_type in Monitoring.components and daemon_id: data_dir = get_data_dir(fsid, daemon_type, daemon_id) if daemon_type == 'prometheus': @@ -1647,6 +1711,9 @@ def get_container(fsid, daemon_type, daemon_id, elif daemon_type == CephIscsi.daemon_type: entrypoint = CephIscsi.entrypoint name = '%s.%s' % (daemon_type, daemon_id) + # So the container can modprobe iscsi_target_mod and have write perms + # to configfs we need to make this a privileged container. + privileged = True else: entrypoint = '' name = '' @@ -1788,6 +1855,11 @@ def deploy_daemon_units(fsid, uid, gid, daemon_type, daemon_id, c, if daemon_type == 'osd': # osds have a pre-start step assert osd_fsid + f.write('# Simple OSDs need chown on startup:\n') + for n in ['block', 'block.db', 'block.wal']: + p = os.path.join(data_dir, n) + f.write('[ ! -L {p} ] || chown {uid}:{gid} {p}\n'.format(p=p, uid=uid, gid=gid)) + f.write('# LVM OSDs use ceph-volume lvm activate:\n') prestart = CephContainer( image=args.image, entrypoint='/usr/sbin/ceph-volume', @@ -2020,7 +2092,7 @@ KillMode=none Restart=on-failure RestartSec=10s TimeoutStartSec=120 -TimeoutStopSec=15 +TimeoutStopSec=120 StartLimitInterval=30min StartLimitBurst=5 @@ -2088,6 +2160,7 @@ class CephContainer: 'run', '--rm', '--net=host', + '--ipc=host', ] + self.container_args + priv + \ cname + envs + \ vols + entrypoint + \ @@ -2121,6 +2194,7 @@ class CephContainer: 'run', '--rm', '--net=host', + '--ipc=host', ] + self.container_args + priv + envs + vols + [ '--entrypoint', cmd[0], self.image @@ -2484,9 +2558,13 @@ def command_bootstrap(): def is_mgr_available(): # type: () -> bool timeout=args.timeout if args.timeout else 30 # seconds - out = cli(['status', '-f', 'json-pretty'], timeout=timeout) - j = json.loads(out) - return j.get('mgrmap', {}).get('available', False) + try: + out = cli(['status', '-f', 'json-pretty'], timeout=timeout) + j = json.loads(out) + return j.get('mgrmap', {}).get('available', False) + except Exception as e: + logger.debug('status failed: %s' % e) + return False is_available('mgr', is_mgr_available) # wait for mgr to restart (after enabling a module) @@ -2517,31 +2595,47 @@ def command_bootstrap(): logger.info('Setting orchestrator backend to cephadm...') cli(['orch', 'set', 'backend', 'cephadm']) - logger.info('Generating ssh key...') - cli(['cephadm', 'generate-key']) - ssh_pub = cli(['cephadm', 'get-pub-key']) - - with open(args.output_pub_ssh_key, 'w') as f: - f.write(ssh_pub) - logger.info('Wrote public SSH key to to %s' % args.output_pub_ssh_key) - - logger.info('Adding key to root@localhost\'s authorized_keys...') - if not os.path.exists('/root/.ssh'): - os.mkdir('/root/.ssh', 0o700) - auth_keys_file = '/root/.ssh/authorized_keys' - add_newline = False - if os.path.exists(auth_keys_file): - with open(auth_keys_file, 'r') as f: - f.seek(0, os.SEEK_END) - if f.tell() > 0: - f.seek(f.tell()-1, os.SEEK_SET) # go to last char - if f.read() != '\n': - add_newline = True - with open(auth_keys_file, 'a') as f: - os.fchmod(f.fileno(), 0o600) # just in case we created it - if add_newline: - f.write('\n') - f.write(ssh_pub.strip() + '\n') + if args.ssh_config: + logger.info('Using provided ssh config...') + mounts = { + pathify(args.ssh_config.name): '/tmp/cephadm-ssh-config:z', + } + cli(['cephadm', 'set-ssh-config', '-i', '/tmp/cephadm-ssh-config'], extra_mounts=mounts) + + if args.ssh_private_key and args.ssh_public_key: + logger.info('Using provided ssh keys...') + mounts = { + pathify(args.ssh_private_key.name): '/tmp/cephadm-ssh-key:z', + pathify(args.ssh_public_key.name): '/tmp/cephadm-ssh-key.pub:z' + } + cli(['cephadm', 'set-priv-key', '-i', '/tmp/cephadm-ssh-key'], extra_mounts=mounts) + cli(['cephadm', 'set-pub-key', '-i', '/tmp/cephadm-ssh-key.pub'], extra_mounts=mounts) + else: + logger.info('Generating ssh key...') + cli(['cephadm', 'generate-key']) + ssh_pub = cli(['cephadm', 'get-pub-key']) + + with open(args.output_pub_ssh_key, 'w') as f: + f.write(ssh_pub) + logger.info('Wrote public SSH key to to %s' % args.output_pub_ssh_key) + + logger.info('Adding key to root@localhost\'s authorized_keys...') + if not os.path.exists('/root/.ssh'): + os.mkdir('/root/.ssh', 0o700) + auth_keys_file = '/root/.ssh/authorized_keys' + add_newline = False + if os.path.exists(auth_keys_file): + with open(auth_keys_file, 'r') as f: + f.seek(0, os.SEEK_END) + if f.tell() > 0: + f.seek(f.tell()-1, os.SEEK_SET) # go to last char + if f.read() != '\n': + add_newline = True + with open(auth_keys_file, 'a') as f: + os.fchmod(f.fileno(), 0o600) # just in case we created it + if add_newline: + f.write('\n') + f.write(ssh_pub.strip() + '\n') host = get_hostname() logger.info('Adding host %s...' % host) @@ -2567,9 +2661,10 @@ def command_bootstrap(): # dashboard crt and key if args.dashboard_key and args.dashboard_crt: logger.info('Using provided dashboard certificate...') - mounts = {} - mounts[pathify(args.dashboard_crt)] = '/tmp/dashboard.crt:z' - mounts[pathify(args.dashboard_key)] = '/tmp/dashboard.key:z' + mounts = { + pathify(args.dashboard_crt.name): '/tmp/dashboard.crt:z', + pathify(args.dashboard_key.name): '/tmp/dashboard.key:z' + } cli(['dashboard', 'set-ssl-certificate', '-i', '/tmp/dashboard.crt'], extra_mounts=mounts) cli(['dashboard', 'set-ssl-certificate-key', '-i', '/tmp/dashboard.key'], extra_mounts=mounts) else: @@ -2593,6 +2688,28 @@ def command_bootstrap(): get_fqdn(), port, args.initial_dashboard_user, password)) + + if args.apply_spec: + logger.info('Applying %s to cluster' % args.apply_spec) + + with open(args.apply_spec) as f: + for line in f: + if 'hostname:' in line: + line = line.replace('\n', '') + split = line.split(': ') + if split[1] != host: + logger.info('Adding ssh key to %s' % split[1]) + + ssh_key = '/etc/ceph/ceph.pub' + if args.ssh_public_key: + ssh_key = args.ssh_public_key.name + out, err, code = call_throws(['ssh-copy-id', '-f', '-i', ssh_key, 'root@%s' % split[1]]) + + mounts = {} + mounts[pathify(args.apply_spec)] = '/tmp/spec.yml:z' + + out = cli(['orch', 'apply', '-i', '/tmp/spec.yml'], extra_mounts=mounts) + logger.info(out) logger.info('You can access the Ceph CLI with:\n\n' '\tsudo %s shell --fsid %s -c %s -k %s\n' % ( @@ -2628,7 +2745,7 @@ def extract_uid_gid_monitoring(daemon_type): @default_image def command_deploy(): # type: () -> None - (daemon_type, daemon_id) = args.name.split('.', 1) + daemon_type, daemon_id = args.name.split('.', 1) l = FileLock(args.fsid) l.acquire() @@ -2636,11 +2753,22 @@ def command_deploy(): if daemon_type not in get_supported_daemons(): raise Error('daemon type %s not recognized' % daemon_type) - logger.info('Deploying daemon %s.%s ...' % (daemon_type, daemon_id)) + redeploy = False + unit_name = get_unit_name(args.fsid, daemon_type, daemon_id) + (_, state, _) = check_unit(unit_name) + if state == 'running': + redeploy = True + + if args.reconfig: + logger.info('%s daemon %s ...' % ('Reconfig', args.name)) + elif redeploy: + logger.info('%s daemon %s ...' % ('Redeploy', args.name)) + else: + logger.info('%s daemon %s ...' % ('Deploy', args.name)) if daemon_type in Ceph.daemons: - (config, keyring) = get_config_and_keyring() - (uid, gid) = extract_uid_gid() + config, keyring = get_config_and_keyring() + uid, gid = extract_uid_gid() make_var_run(args.fsid, uid, gid) c = get_container(args.fsid, daemon_type, daemon_id, ptrace=args.allow_ptrace) @@ -2651,10 +2779,8 @@ def command_deploy(): elif daemon_type in Monitoring.components: # monitoring daemon - prometheus, grafana, alertmanager, node-exporter - monitoring_args = [] # type: List[str] - # Default Checks - if not args.reconfig: + if not args.reconfig and not redeploy: daemon_ports = Monitoring.port_map[daemon_type] # type: List[int] if any([port_in_use(port) for port in daemon_ports]): raise Error("TCP Port(s) '{}' required for {} is already in use".format(",".join(map(str, daemon_ports)), daemon_type)) @@ -2672,24 +2798,25 @@ def command_deploy(): raise Error("{} deployment requires config-json which must " "contain arg for {}".format(daemon_type.capitalize(), ', '.join(required_args))) - uid, gid = extract_uid_gid_monitoring(daemon_type) c = get_container(args.fsid, daemon_type, daemon_id) deploy_daemon(args.fsid, daemon_type, daemon_id, c, uid, gid, reconfig=args.reconfig) elif daemon_type == NFSGanesha.daemon_type: - NFSGanesha.port_in_use() - (config, keyring) = get_config_and_keyring() + if not args.reconfig and not redeploy: + NFSGanesha.port_in_use() + config, keyring = get_config_and_keyring() # TODO: extract ganesha uid/gid (997, 994) ? - (uid, gid) = extract_uid_gid() + uid, gid = extract_uid_gid() c = get_container(args.fsid, daemon_type, daemon_id) deploy_daemon(args.fsid, daemon_type, daemon_id, c, uid, gid, config=config, keyring=keyring, reconfig=args.reconfig) + elif daemon_type == CephIscsi.daemon_type: - (config, keyring) = get_config_and_keyring() - (uid, gid) = extract_uid_gid() + config, keyring = get_config_and_keyring() + uid, gid = extract_uid_gid() c = get_container(args.fsid, daemon_type, daemon_id) deploy_daemon(args.fsid, daemon_type, daemon_id, c, uid, gid, config=config, keyring=keyring, @@ -2710,6 +2837,7 @@ def command_run(): ################################## @infer_fsid +@infer_config @infer_image def command_shell(): # type: () -> int @@ -2731,8 +2859,6 @@ def command_shell(): # use /etc/ceph files by default, if present. we do this instead of # making these defaults in the arg parser because we don't want an error # if they don't exist. - if not args.config and os.path.exists(SHELL_DEFAULT_CONF): - args.config = SHELL_DEFAULT_CONF if not args.keyring and os.path.exists(SHELL_DEFAULT_KEYRING): args.keyring = SHELL_DEFAULT_KEYRING @@ -2743,6 +2869,10 @@ def command_shell(): mounts[pathify(args.config)] = '/etc/ceph/ceph.conf:z' if args.keyring: mounts[pathify(args.keyring)] = '/etc/ceph/ceph.keyring:z' + if args.mount: + mount = pathify(args.mount) + filename = os.path.basename(mount) + mounts[mount] = '/mnt/{}:z'.format(filename) if args.command: command = args.command else: @@ -2836,6 +2966,7 @@ def command_ceph_volume(): c = CephContainer( image=args.image, entrypoint='/usr/sbin/ceph-volume', + envs=args.env, args=args.command, privileged=True, volume_mounts=mounts, @@ -2851,8 +2982,9 @@ def command_unit(): # type: () -> None if not args.fsid: raise Error('must pass --fsid to specify cluster') - (daemon_type, daemon_id) = args.name.split('.', 1) - unit_name = get_unit_name(args.fsid, daemon_type, daemon_id) + + unit_name = get_unit_name_by_daemon_name(args.fsid, args.name) + call_throws([ 'systemctl', args.command, @@ -2866,8 +2998,7 @@ def command_logs(): if not args.fsid: raise Error('must pass --fsid to specify cluster') - (daemon_type, daemon_id) = args.name.split('.', 1) - unit_name = get_unit_name(args.fsid, daemon_type, daemon_id) + unit_name = get_unit_name_by_daemon_name(args.fsid, args.name) cmd = [find_program('journalctl')] cmd.extend(['-u', unit_name]) @@ -2944,14 +3075,15 @@ def list_daemons(detail=True, legacy_dir=None): fsid = get_legacy_daemon_fsid( cluster, daemon_type, daemon_id, legacy_dir=legacy_dir) + legacy_unit_name = 'ceph-%s@%s' % (daemon_type, daemon_id) i = { 'style': 'legacy', 'name': '%s.%s' % (daemon_type, daemon_id), 'fsid': fsid if fsid is not None else 'unknown', + 'systemd_unit': legacy_unit_name, } if detail: - (i['enabled'], i['state'], _) = check_unit( - 'ceph-%s@%s' % (daemon_type, daemon_id)) + (i['enabled'], i['state'], _) = check_unit(legacy_unit_name) if not host_version: try: out, err, code = call(['ceph', '-v']) @@ -2976,6 +3108,7 @@ def list_daemons(detail=True, legacy_dir=None): 'style': 'cephadm:v1', 'name': name, 'fsid': fsid, + 'systemd_unit': unit_name, } if detail: # get container id @@ -3062,11 +3195,21 @@ def list_daemons(detail=True, legacy_dir=None): ls.append(i) - # /var/lib/rook - # WRITE ME return ls +def get_daemon_description(fsid, name, detail=False, legacy_dir=None): + # type: (str, str, bool, Optional[str]) -> Dict[str, str] + + for d in list_daemons(detail=detail, legacy_dir=legacy_dir): + if d['fsid'] != fsid: + continue + if d['name'] != name: + continue + return d + raise Error('Daemon not found: {}. See `cephadm ls`'.format(name)) + + ################################## @default_image @@ -3124,13 +3267,13 @@ class AdoptOsd(object): with open(path, 'r') as f: osd_fsid = f.read().strip() logger.info("Found online OSD at %s" % path) - if os.path.exists(os.path.join(self.osd_data_dir, 'type')): - with open(os.path.join(self.osd_data_dir, 'type')) as f: - osd_type = f.read().strip() - else: - logger.info('"type" file missing for OSD data dir') except IOError: logger.info('Unable to read OSD fsid from %s' % path) + if os.path.exists(os.path.join(self.osd_data_dir, 'type')): + with open(os.path.join(self.osd_data_dir, 'type')) as f: + osd_type = f.read().strip() + else: + logger.info('"type" file missing for OSD data dir') return osd_fsid, osd_type @@ -3212,6 +3355,7 @@ def command_adopt_ceph(daemon_type, daemon_id, fsid): if not osd_fsid: raise Error('Unable to find OSD {}'.format(daemon_id)) logger.info('objectstore_type is %s' % osd_type) + assert osd_type if osd_type == 'filestore': raise Error('FileStore is not supported by cephadm') @@ -3464,11 +3608,13 @@ def command_rm_daemon(): l = FileLock(args.fsid) l.acquire() + unit_name = get_unit_name_by_daemon_name(args.fsid, args.name) + (daemon_type, daemon_id) = args.name.split('.', 1) if daemon_type in ['mon', 'osd'] and not args.force: raise Error('must pass --force to proceed: ' 'this command may destroy precious data!') - unit_name = get_unit_name(args.fsid, daemon_type, daemon_id) + call(['systemctl', 'stop', unit_name], verbose_on_failure=False) call(['systemctl', 'reset-failed', unit_name], @@ -3569,7 +3715,7 @@ def check_time_sync(enabler=None): 'ntpd.service', # el7 (at least) 'ntp.service', # 18.04 (at least) ] - if not check_units(units, enabler=None): + if not check_units(units, enabler): logger.warning('No time sync service is running; checked for %s' % units) return False return True @@ -3676,6 +3822,7 @@ class CustomValidation(argparse.Action): ################################## def get_distro(): + # type: () -> Tuple[Optional[str], Optional[str], Optional[str]] distro = None distro_version = None distro_codename = None @@ -4140,6 +4287,11 @@ def _get_parser(): type=int, default=DEFAULT_RETRY, help='max number of retries') + parser.add_argument( + '--env', '-e', + action='append', + default=[], + help='set environment variable') subparsers = parser.add_subparsers(help='sub-command') @@ -4266,13 +4418,16 @@ def _get_parser(): parser_shell.add_argument( '--keyring', '-k', help='ceph.keyring to pass through to the container') + parser_shell.add_argument( + '--mount', '-m', + help='file or directory path that will be mounted in container /mnt') parser_shell.add_argument( '--env', '-e', action='append', default=[], help='set environment variable') parser_shell.add_argument( - 'command', nargs='*', + 'command', nargs=argparse.REMAINDER, help='command (optional)') parser_enter = subparsers.add_parser( @@ -4286,7 +4441,7 @@ def _get_parser(): required=True, help='daemon name (type.id)') parser_enter.add_argument( - 'command', nargs='*', + 'command', nargs=argparse.REMAINDER, help='command') parser_ceph_volume = subparsers.add_parser( @@ -4305,7 +4460,7 @@ def _get_parser(): '--keyring', '-k', help='ceph.keyring to pass through to the container') parser_ceph_volume.add_argument( - 'command', nargs='+', + 'command', nargs=argparse.REMAINDER, help='command') parser_unit = subparsers.add_parser( @@ -4386,11 +4541,26 @@ def _get_parser(): parser_bootstrap.add_argument( '--dashboard-key', + type=argparse.FileType('r'), help='Dashboard key') parser_bootstrap.add_argument( '--dashboard-crt', + type=argparse.FileType('r'), help='Dashboard certificate') + parser_bootstrap.add_argument( + '--ssh-config', + type=argparse.FileType('r'), + help='SSH config') + parser_bootstrap.add_argument( + '--ssh-private-key', + type=argparse.FileType('r'), + help='SSH private key') + parser_bootstrap.add_argument( + '--ssh-public-key', + type=argparse.FileType('r'), + help='SSH public key') + parser_bootstrap.add_argument( '--skip-mon-network', action='store_true', @@ -4439,6 +4609,15 @@ def _get_parser(): '--skip-monitoring-stack', action='store_true', help='Do not automatically provision monitoring stack (prometheus, grafana, alertmanager, node-exporter)') + parser_bootstrap.add_argument( + '--apply-spec', + help='Apply cluster spec after bootstrap (copy ssh key, add hosts and apply services)') + + + parser_bootstrap.add_argument( + '--shared_ceph_folder', + metavar='CEPH_SOURCE_FOLDER', + help='Development mode. Several folders in containers are volumes mapped to different sub-folders in the ceph source folder') parser_deploy = subparsers.add_parser( 'deploy', help='deploy a daemon') @@ -4534,7 +4713,10 @@ def _get_parser(): def _parse_args(av): parser = _get_parser() - return parser.parse_args(av) + args = parser.parse_args(av) + if 'command' in args and args.command and args.command[0] == "--": + args.command.pop(0) + return args if __name__ == "__main__": # allow argv to be injected diff --git a/ceph/src/cephadm/mypy.ini b/ceph/src/cephadm/mypy.ini index 1215375ed..313abc807 100644 --- a/ceph/src/cephadm/mypy.ini +++ b/ceph/src/cephadm/mypy.ini @@ -1,2 +1,2 @@ [mypy] -ignore_missing_imports = True \ No newline at end of file +ignore_missing_imports = False diff --git a/ceph/src/cephadm/tests/test_cephadm.py b/ceph/src/cephadm/tests/test_cephadm.py index 6f51b3e78..785aedac7 100644 --- a/ceph/src/cephadm/tests/test_cephadm.py +++ b/ceph/src/cephadm/tests/test_cephadm.py @@ -1,3 +1,4 @@ +# type: ignore import argparse import mock import os diff --git a/ceph/src/client/Client.cc b/ceph/src/client/Client.cc index d3dceee2e..c84da997a 100644 --- a/ceph/src/client/Client.cc +++ b/ceph/src/client/Client.cc @@ -96,7 +96,7 @@ #include "include/ceph_assert.h" #include "include/stat.h" -#include "include/cephfs/ceph_statx.h" +#include "include/cephfs/ceph_ll_client.h" #if HAVE_GETGROUPLIST #include @@ -270,6 +270,7 @@ Client::Client(Messenger *m, MonClient *mc, Objecter *objecter_) async_dentry_invalidator(m->cct), interrupt_finisher(m->cct), remount_finisher(m->cct), + async_ino_releasor(m->cct), objecter_finisher(m->cct), m_command_hook(this), fscid(0) @@ -303,9 +304,6 @@ Client::Client(Messenger *m, MonClient *mc, Objecter *objecter_) cct->_conf->client_oc_target_dirty, cct->_conf->client_oc_max_dirty_age, true)); - objecter_finisher.start(); - filer.reset(new Filer(objecter, &objecter_finisher)); - objecter->enable_blacklist_events(); } @@ -473,10 +471,20 @@ void Client::dump_status(Formatter *f) } } -int Client::init() +void Client::_pre_init() { timer.init(); + + objecter_finisher.start(); + filer.reset(new Filer(objecter, &objecter_finisher)); + objecter->enable_blacklist_events(); + objectcacher->start(); +} + +int Client::init() +{ + _pre_init(); { std::lock_guard l{client_lock}; ceph_assert(!initialized); @@ -582,6 +590,12 @@ void Client::shutdown() remount_finisher.stop(); } + if (ino_release_cb) { + ldout(cct, 10) << "shutdown stopping inode release finisher" << dendl; + async_ino_releasor.wait_for_empty(); + async_ino_releasor.stop(); + } + objectcacher->stop(); // outside of client_lock! this does a join. { std::lock_guard l{client_lock}; @@ -3623,15 +3637,17 @@ void Client::check_caps(Inode *in, unsigned flags) } int flushing; + int msg_flags = 0; ceph_tid_t flush_tid; if (in->auth_cap == &cap && in->dirty_caps) { flushing = mark_caps_flushing(in, &flush_tid); + if (flags & CHECK_CAPS_SYNCHRONOUS) + msg_flags |= MClientCaps::FLAG_SYNC; } else { flushing = 0; flush_tid = 0; } - int msg_flags = (flags & CHECK_CAPS_SYNCHRONOUS) ? MClientCaps::FLAG_SYNC : 0; send_cap(in, session, &cap, msg_flags, cap_used, wanted, retain, flushing, flush_tid); } @@ -4265,6 +4281,39 @@ void Client::_trim_negative_child_dentries(InodeRef& in) } } +class C_Client_CacheRelease : public Context { +private: + Client *client; + vinodeno_t ino; +public: + C_Client_CacheRelease(Client *c, Inode *in) : + client(c) { + if (client->use_faked_inos()) + ino = vinodeno_t(in->faked_ino, CEPH_NOSNAP); + else + ino = in->vino(); + } + void finish(int r) override { + ceph_assert(ceph_mutex_is_not_locked_by_me(client->client_lock)); + client->_async_inode_release(ino); + } +}; + +void Client::_async_inode_release(vinodeno_t ino) +{ + if (unmounting) + return; + ldout(cct, 10) << __func__ << " " << ino << dendl; + ino_release_cb(callback_handle, ino); +} + +void Client::_schedule_ino_release_callback(Inode *in) { + + if (ino_release_cb) + // we queue the invalidate, which calls the callback and decrements the ref + async_ino_releasor.queue(new C_Client_CacheRelease(this, in)); +} + void Client::trim_caps(MetaSession *s, uint64_t max) { mds_rank_t mds = s->mds_num; @@ -4319,6 +4368,7 @@ void Client::trim_caps(MetaSession *s, uint64_t max) if (all && in->ino != MDS_INO_ROOT) { ldout(cct, 20) << __func__ << " counting as trimmed: " << *in << dendl; trimmed++; + _schedule_ino_release_callback(in.get()); } } } @@ -5135,7 +5185,7 @@ void Client::_async_dentry_invalidate(vinodeno_t dirino, vinodeno_t ino, string& return; ldout(cct, 10) << __func__ << " '" << name << "' ino " << ino << " in dir " << dirino << dendl; - dentry_invalidate_cb(callback_handle, dirino, ino, name); + dentry_invalidate_cb(callback_handle, dirino, ino, name.c_str(), name.length()); } void Client::_schedule_invalidate_dentry_callback(Dentry *dn, bool del) @@ -10420,7 +10470,7 @@ int Client::ll_statfs(Inode *in, struct statvfs *stbuf, const UserPerm& perms) return statfs(0, stbuf, perms); } -void Client::ll_register_callbacks(struct client_callback_args *args) +void Client::ll_register_callbacks(struct ceph_client_callback_args *args) { if (!args) return; @@ -10448,7 +10498,12 @@ void Client::ll_register_callbacks(struct client_callback_args *args) remount_cb = args->remount_cb; remount_finisher.start(); } - umask_cb = args->umask_cb; + if (args->ino_release_cb) { + ino_release_cb = args->ino_release_cb; + async_ino_releasor.start(); + } + if (args->umask_cb) + umask_cb = args->umask_cb; } int Client::test_dentry_handling(bool can_invalidate) @@ -14623,8 +14678,7 @@ StandaloneClient::~StandaloneClient() int StandaloneClient::init() { - timer.init(); - objectcacher->start(); + _pre_init(); objecter->init(); client_lock.lock(); diff --git a/ceph/src/client/Client.h b/ceph/src/client/Client.h index 0ec989db2..a927d85b1 100644 --- a/ceph/src/client/Client.h +++ b/ceph/src/client/Client.h @@ -23,7 +23,7 @@ #include "common/cmdparse.h" #include "common/compiler_extensions.h" #include "include/common_fwd.h" -#include "include/cephfs/ceph_statx.h" +#include "include/cephfs/ceph_ll_client.h" #include "include/filepath.h" #include "include/interval_set.h" #include "include/lru.h" @@ -121,28 +121,6 @@ struct CapSnap; struct MetaRequest; class ceph_lock_state_t; - -typedef void (*client_ino_callback_t)(void *handle, vinodeno_t ino, int64_t off, int64_t len); - -typedef void (*client_dentry_callback_t)(void *handle, vinodeno_t dirino, - vinodeno_t ino, string& name); -typedef int (*client_remount_callback_t)(void *handle); - -typedef void(*client_switch_interrupt_callback_t)(void *handle, void *data); -typedef mode_t (*client_umask_callback_t)(void *handle); - -/* Callback for delegation recalls */ -typedef void (*ceph_deleg_cb_t)(Fh *fh, void *priv); - -struct client_callback_args { - void *handle; - client_ino_callback_t ino_cb; - client_dentry_callback_t dentry_cb; - client_switch_interrupt_callback_t switch_intr_cb; - client_remount_callback_t remount_cb; - client_umask_callback_t umask_cb; -}; - // ======================================================== // client interface @@ -254,6 +232,7 @@ public: friend class C_Client_Remount; friend class C_Client_RequestInterrupt; friend class C_Deleg_Timeout; // Asserts on client_lock, called when a delegation is unreturned + friend class C_Client_CacheRelease; // Asserts on client_lock friend class SyntheticClient; friend void intrusive_ptr_release(Inode *in); @@ -601,7 +580,7 @@ public: int ll_osdaddr(int osd, uint32_t *addr); int ll_osdaddr(int osd, char* buf, size_t size); - void ll_register_callbacks(struct client_callback_args *args); + void ll_register_callbacks(struct ceph_client_callback_args *args); int test_dentry_handling(bool can_invalidate); const char** get_tracked_conf_keys() const override; @@ -695,6 +674,10 @@ public: void _invalidate_inode_cache(Inode *in); void _invalidate_inode_cache(Inode *in, int64_t off, int64_t len); void _async_invalidate(vinodeno_t ino, int64_t off, int64_t len); + + void _schedule_ino_release_callback(Inode *in); + void _async_inode_release(vinodeno_t ino); + bool _release(Inode *in); /** @@ -953,6 +936,8 @@ protected: void _close_sessions(); + void _pre_init(); + /** * The basic housekeeping parts of init (perf counters, admin socket) * that is independent of how objecters/monclient/messengers are @@ -1204,6 +1189,7 @@ private: client_ino_callback_t ino_invalidate_cb = nullptr; client_dentry_callback_t dentry_invalidate_cb = nullptr; client_umask_callback_t umask_cb = nullptr; + client_ino_release_t ino_release_cb = nullptr; void *callback_handle = nullptr; bool can_invalidate_dentries = false; @@ -1211,6 +1197,7 @@ private: Finisher async_dentry_invalidator; Finisher interrupt_finisher; Finisher remount_finisher; + Finisher async_ino_releasor; Finisher objecter_finisher; Context *tick_event = nullptr; diff --git a/ceph/src/client/Delegation.h b/ceph/src/client/Delegation.h index e260f6c99..d24a02487 100644 --- a/ceph/src/client/Delegation.h +++ b/ceph/src/client/Delegation.h @@ -5,8 +5,7 @@ #include "common/Clock.h" #include "common/Timer.h" - -class Fh; +#include "include/cephfs/ceph_ll_client.h" /* Commands for manipulating delegation state */ #ifndef CEPH_DELEGATION_NONE @@ -15,9 +14,6 @@ class Fh; # define CEPH_DELEGATION_WR 2 #endif -/* Callback for delegation recalls */ -typedef void (*ceph_deleg_cb_t)(Fh *fh, void *priv); - /* Converts CEPH_DELEGATION_* to cap mask */ int ceph_deleg_caps_for_type(unsigned type); diff --git a/ceph/src/client/SyntheticClient.cc b/ceph/src/client/SyntheticClient.cc index a6d05ff3e..9ef5ccf19 100644 --- a/ceph/src/client/SyntheticClient.cc +++ b/ceph/src/client/SyntheticClient.cc @@ -36,7 +36,7 @@ #include "common/errno.h" #include "include/ceph_assert.h" -#include "include/cephfs/ceph_statx.h" +#include "include/cephfs/ceph_ll_client.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_client diff --git a/ceph/src/client/fuse_ll.cc b/ceph/src/client/fuse_ll.cc index 3aeda459c..82e979ffe 100644 --- a/ceph/src/client/fuse_ll.cc +++ b/ceph/src/client/fuse_ll.cc @@ -33,10 +33,10 @@ #include "ioctl.h" #include "common/config.h" #include "include/ceph_assert.h" -#include "include/cephfs/ceph_statx.h" +#include "include/cephfs/ceph_ll_client.h" +#include "include/ceph_fuse.h" #include "fuse_ll.h" -#include #include #define dout_context g_ceph_context @@ -640,7 +640,13 @@ static void fuse_ll_flush(fuse_req_t req, fuse_ino_t ino, } #ifdef FUSE_IOCTL_COMPAT -static void fuse_ll_ioctl(fuse_req_t req, fuse_ino_t ino, int cmd, void *arg, struct fuse_file_info *fi, +static void fuse_ll_ioctl(fuse_req_t req, fuse_ino_t ino, +#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 5) + unsigned int cmd, +#else + int cmd, +#endif + void *arg, struct fuse_file_info *fi, unsigned flags, const void *in_buf, size_t in_bufsz, size_t out_bufsz) { CephFuse::Handle *cfuse = fuse_ll_req_prepare(req); @@ -937,7 +943,7 @@ static void ino_invalidate_cb(void *handle, vinodeno_t vino, int64_t off, } static void dentry_invalidate_cb(void *handle, vinodeno_t dirino, - vinodeno_t ino, string& name) + vinodeno_t ino, const char *name, size_t len) { CephFuse::Handle *cfuse = (CephFuse::Handle *)handle; fuse_ino_t fdirino = cfuse->make_fake_ino(dirino.ino, dirino.snapid); @@ -946,12 +952,12 @@ static void dentry_invalidate_cb(void *handle, vinodeno_t dirino, if (ino.ino != inodeno_t()) fino = cfuse->make_fake_ino(ino.ino, ino.snapid); #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0) - fuse_lowlevel_notify_delete(cfuse->se, fdirino, fino, name.c_str(), name.length()); + fuse_lowlevel_notify_delete(cfuse->se, fdirino, fino, name, len); #else - fuse_lowlevel_notify_delete(cfuse->ch, fdirino, fino, name.c_str(), name.length()); + fuse_lowlevel_notify_delete(cfuse->ch, fdirino, fino, name, len); #endif #elif FUSE_VERSION >= FUSE_MAKE_VERSION(2, 8) - fuse_lowlevel_notify_inval_entry(cfuse->ch, fdirino, name.c_str(), name.length()); + fuse_lowlevel_notify_inval_entry(cfuse->ch, fdirino, name, len); #endif } @@ -1233,7 +1239,7 @@ int CephFuse::Handle::start() #endif - struct client_callback_args args = { + struct ceph_client_callback_args args = { handle: this, ino_cb: client->cct->_conf.get_val("fuse_use_invalidate_cb") ? ino_invalidate_cb : NULL, @@ -1256,7 +1262,14 @@ int CephFuse::Handle::loop() auto fuse_multithreaded = client->cct->_conf.get_val( "fuse_multithreaded"); if (fuse_multithreaded) { -#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0) +#if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 1) + { + struct fuse_loop_config conf = { 0 }; + + conf.clone_fd = opts.clone_fd; + return fuse_session_loop_mt(se, &conf); + } +#elif FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0) return fuse_session_loop_mt(se, opts.clone_fd); #else return fuse_session_loop_mt(se); diff --git a/ceph/src/client/fuse_ll.h b/ceph/src/client/fuse_ll.h index 3d7b891dd..85e587baf 100644 --- a/ceph/src/client/fuse_ll.h +++ b/ceph/src/client/fuse_ll.h @@ -12,8 +12,6 @@ * */ -#define FUSE_USE_VERSION 30 - class CephFuse { public: CephFuse(Client *c, int fd); diff --git a/ceph/src/cls/queue/cls_queue_src.cc b/ceph/src/cls/queue/cls_queue_src.cc index b48dcd19f..6fcfc6288 100644 --- a/ceph/src/cls/queue/cls_queue_src.cc +++ b/ceph/src/cls/queue/cls_queue_src.cc @@ -25,6 +25,11 @@ int queue_write_head(cls_method_context_t hctx, cls_queue_head& head) bl.claim_append(bl_head); + if (bl.length() > head.max_head_size) { + CLS_LOG(0, "ERROR: queue_write_head: invalid head size = %u and urgent data size = %u \n", bl.length(), head.bl_urgent_data.length()); + return -EINVAL; + } + int ret = cls_cxx_write2(hctx, 0, bl.length(), &bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); if (ret < 0) { CLS_LOG(5, "ERROR: queue_write_head: failed to write head\n"); @@ -266,6 +271,7 @@ int queue_list_entries(cls_method_context_t hctx, const cls_queue_list_op& op, c uint64_t data_size = 0, num_ops = 0; uint16_t entry_start = 0; bufferlist bl; + string last_marker; do { CLS_LOG(10, "INFO: queue_list_entries(): start_offset is %lu\n", start_offset); @@ -306,6 +312,10 @@ int queue_list_entries(cls_method_context_t hctx, const cls_queue_list_op& op, c CLS_LOG(10, "INFO: queue_list_entries(): index: %u, size_to_process: %lu\n", index, size_to_process); cls_queue_entry entry; ceph_assert(it.get_off() == index); + //Use the last marker saved in previous iteration as the marker for this entry + if (offset_populated) { + entry.marker = last_marker; + } //Populate offset if not done in previous iteration if (! offset_populated) { cls_queue_marker marker = {entry_start_offset + index, gen}; @@ -339,6 +349,7 @@ int queue_list_entries(cls_method_context_t hctx, const cls_queue_list_op& op, c // Copy unprocessed data to bl bl_chunk.splice(index, size_to_process, &bl); offset_populated = true; + last_marker = entry.marker; CLS_LOG(10, "INFO: queue_list_entries: not enough data to read entry start and data size, breaking out!\n"); break; } @@ -355,6 +366,7 @@ int queue_list_entries(cls_method_context_t hctx, const cls_queue_list_op& op, c it.copy(size_to_process, bl); offset_populated = true; entry_start_processed = true; + last_marker = entry.marker; CLS_LOG(10, "INFO: queue_list_entries(): not enough data to read data, breaking out!\n"); break; } @@ -365,6 +377,7 @@ int queue_list_entries(cls_method_context_t hctx, const cls_queue_list_op& op, c data_size = 0; entry_start = 0; num_ops++; + last_marker.clear(); if (num_ops == op.max) { CLS_LOG(10, "INFO: queue_list_entries(): num_ops is same as op.max, hence breaking out from inner loop!\n"); break; diff --git a/ceph/src/cls/rgw_gc/cls_rgw_gc.cc b/ceph/src/cls/rgw_gc/cls_rgw_gc.cc index 976e49ea5..f45fb2df2 100644 --- a/ceph/src/cls/rgw_gc/cls_rgw_gc.cc +++ b/ceph/src/cls/rgw_gc/cls_rgw_gc.cc @@ -361,7 +361,9 @@ static int cls_rgw_gc_queue_remove_entries(cls_method_context_t hctx, bufferlist } //Update urgent data map + head.bl_urgent_data.clear(); encode(urgent_data, head.bl_urgent_data); + CLS_LOG(5, "INFO: cls_rgw_gc_queue_remove_entries(): Urgent data size is %u\n", head.bl_urgent_data.length()); return queue_write_head(hctx, head); } diff --git a/ceph/src/common/config.cc b/ceph/src/common/config.cc index eb6b721e1..dd4958d40 100644 --- a/ceph/src/common/config.cc +++ b/ceph/src/common/config.cc @@ -659,6 +659,7 @@ int md_config_t::parse_argv(ConfigValues& values, set_val_or_die(values, tracker, "daemonize", "false"); } else if (ceph_argparse_flag(args, i, "-d", (char*)NULL)) { + set_val_or_die(values, tracker, "fuse_debug", "true"); set_val_or_die(values, tracker, "daemonize", "false"); set_val_or_die(values, tracker, "log_file", ""); set_val_or_die(values, tracker, "log_to_stderr", "true"); diff --git a/ceph/src/common/legacy_config_opts.h b/ceph/src/common/legacy_config_opts.h index bee70445d..b0562247d 100644 --- a/ceph/src/common/legacy_config_opts.h +++ b/ceph/src/common/legacy_config_opts.h @@ -1290,6 +1290,8 @@ OPTION(rgw_service_provider_name, OPT_STR) //service provider name which is cont OPTION(rgw_content_length_compat, OPT_BOOL) // Check both HTTP_CONTENT_LENGTH and CONTENT_LENGTH in fcgi env OPTION(rgw_lifecycle_work_time, OPT_STR) //job process lc at 00:00-06:00s OPTION(rgw_lc_lock_max_time, OPT_INT) // total run time for a single lc processor work +OPTION(rgw_lc_max_worker, OPT_INT)// number of (parellized) LCWorker threads +OPTION(rgw_lc_max_wp_worker, OPT_INT)// number of per-LCWorker pool threads OPTION(rgw_lc_max_objs, OPT_INT) OPTION(rgw_lc_max_rules, OPT_U32) // Max rules set on one bucket OPTION(rgw_lc_debug_interval, OPT_INT) // Debug run interval, in seconds diff --git a/ceph/src/common/options.cc b/ceph/src/common/options.cc index f44c03f6c..b9d5b675a 100644 --- a/ceph/src/common/options.cc +++ b/ceph/src/common/options.cc @@ -4004,7 +4004,8 @@ std::vector