From 28e407b858acd3bddc89f68583571f771bb42e46 Mon Sep 17 00:00:00 2001 From: Alwin Antreich Date: Wed, 18 Jul 2018 09:51:46 +0200 Subject: [PATCH] update sources to 12.2.7 Signed-off-by: Alwin Antreich --- Makefile | 4 +- ceph/CMakeLists.txt | 2 +- ceph/alpine/APKBUILD | 6 +- ceph/ceph.spec | 8 +- ceph/ceph.spec.in | 2 + ceph/debian/ceph-osd.postinst | 2 + ceph/debian/changelog | 12 + ceph/doc/_templates/smarttoc.html | 2 +- ceph/doc/ceph-volume/index.rst | 8 +- ceph/doc/ceph-volume/lvm/activate.rst | 27 +- ceph/doc/ceph-volume/lvm/encryption.rst | 9 +- ceph/doc/ceph-volume/lvm/list.rst | 11 + ceph/doc/ceph-volume/lvm/prepare.rst | 7 + ceph/doc/ceph-volume/simple/index.rst | 13 + ceph/doc/cephfs/administration.rst | 16 + ceph/doc/dev/rbd-export.rst | 22 +- ceph/doc/dev/testing.rst | 40 + ceph/doc/man/8/ceph-fuse.rst | 17 +- ceph/doc/man/8/ceph-volume.rst | 8 + ceph/doc/man/8/radosgw-admin.rst | 6 +- ceph/doc/mgr/balancer.rst | 146 ++++ ceph/doc/mgr/index.rst | 1 + ceph/doc/mgr/influx.rst | 3 +- ceph/doc/mgr/prometheus.rst | 84 +-- .../rados/configuration/auth-config-ref.rst | 18 +- ceph/doc/rados/deployment/ceph-deploy-mds.rst | 4 - ceph/doc/rados/operations/add-or-rm-osds.rst | 1 + ceph/doc/rados/operations/pools.rst | 7 + ceph/doc/radosgw/frontends.rst | 18 +- ceph/doc/radosgw/keystone.rst | 7 + ceph/doc/radosgw/multitenancy.rst | 1 + ceph/qa/cephfs/clusters/3-mds.yaml | 2 +- ceph/qa/cephfs/clusters/9-mds.yaml | 2 +- ceph/qa/distros/all/rhel_7.5.yaml | 2 + ceph/qa/distros/all/ubuntu_18.04.yaml | 2 + ceph/qa/overrides/more-active-recovery.yaml | 6 + ceph/qa/standalone/ceph-helpers.sh | 5 + .../erasure-code/test-erasure-code.sh | 6 - .../erasure-code/test-erasure-eio.sh | 127 +++- .../qa/standalone/osd/ec-error-rollforward.sh | 59 ++ ceph/qa/standalone/osd/osd-rep-recov-eio.sh | 6 - ceph/qa/standalone/osd/repro_long_log.sh | 7 +- .../qa/standalone/scrub/osd-recovery-scrub.sh | 1 + ceph/qa/standalone/scrub/osd-scrub-repair.sh | 80 +- ceph/qa/standalone/scrub/osd-scrub-snaps.sh | 682 ++++++++++++++++-- ceph/qa/standalone/scrub/osd-scrub-test.sh | 5 +- .../basic/tasks/ceph-admin-commands.yaml | 14 +- .../ceph-volume/cluster/4node.yaml | 6 + .../tasks/cephfs_scrub_tests.yaml | 2 +- .../tasks/client-recovery.yaml | 1 + .../bugs/client_trim_caps}/% | 0 .../fs/bugs/client_trim_caps/begin.yaml | 1 + .../clusters/small-cluster.yaml | 9 + .../objectstore/bluestore.yaml | 1 + .../fs/bugs/client_trim_caps/overrides/+ | 0 .../client_trim_caps/overrides/debug.yaml | 1 + .../overrides/frag_enable.yaml | 1 + .../overrides/no_client_pidfile.yaml | 1 + .../overrides/whitelist_health.yaml | 1 + .../whitelist_wrongly_marked_down.yaml | 1 + .../client_trim_caps/tasks/trim-i22073.yaml | 20 + .../recovery/tasks/client-recovery.yaml | 1 + .../powercycle/osd/tasks/rados_api_tests.yaml | 1 + .../rados/basic-luminous/scrub_test.yaml | 11 +- .../rados/basic/tasks/rados_api_tests.yaml | 11 +- .../suites/rados/basic/tasks/repair_test.yaml | 6 +- ceph/qa/suites/rados/monthrash/ceph.yaml | 1 - .../multimon/tasks/mon_clock_no_skews.yaml | 2 +- .../suites/rados/objectstore/objectstore.yaml | 4 +- .../singleton-bluestore/all/cephtool.yaml | 18 +- .../all/admin_socket_output.yaml | 8 +- .../all/large-omap-object-warnings.yaml | 22 + .../rados/singleton/all/divergent_priors.yaml | 3 - .../singleton/all/divergent_priors2.yaml | 3 - .../all/osd-recovery-incomplete.yaml | 1 + .../rados/singleton/all/osd-recovery.yaml | 1 + .../all/pg-removal-interruption.yaml | 6 +- .../rados/singleton/all/random-eio.yaml | 4 +- .../rados/singleton/all/thrash-eio.yaml | 8 +- .../thrash_cache_writeback_proxy_none.yaml | 2 +- .../recovery-overrides | 1 + .../recovery-overrides | 1 + .../recovery-overrides | 1 + .../recovery-overrides | 1 + .../thrash-erasure-code/recovery-overrides | 1 + .../rados/thrash/2-recovery-overrides/$ | 0 .../thrash/2-recovery-overrides/default.yaml | 0 .../more-active-recovery.yaml | 1 + .../thrash/workloads/rados_api_tests.yaml | 1 + .../rados/verify/tasks/rados_api_tests.yaml | 1 + .../suites/rbd/basic/msgr-failures/many.yaml | 5 - .../qa/suites/rbd/openstack/base/install.yaml | 2 +- .../qa/suites/rbd/thrash/thrashers/cache.yaml | 4 +- .../smoke/basic/tasks/rados_api_tests.yaml | 2 +- .../luminous-client-x/basic/% | 0 .../luminous-client-x/basic/0-cluster/+ | 0 .../basic/0-cluster/openstack.yaml | 4 + .../basic/0-cluster/start.yaml | 15 + .../basic/1-install/luminous-client-x.yaml | 11 + .../basic/2-workload/rbd_api_tests.yaml | 21 + .../2-workload/rbd_cli_import_export.yaml | 11 + .../basic/supported/centos_7.4.yaml | 1 + .../basic/supported/rhel_7.5.yaml | 1 + .../basic/supported/ubuntu_16.04.yaml | 1 + .../basic/supported/ubuntu_18.04.yaml | 1 + .../luminous-client-x/rbd/% | 0 .../luminous-client-x/rbd/0-cluster/+ | 0 .../rbd/0-cluster/openstack.yaml | 4 + .../rbd/0-cluster/start.yaml | 15 + .../rbd/1-install/luminous-client-x.yaml | 11 + .../rbd/2-features/defaults.yaml | 6 + .../rbd/2-features/layering.yaml | 6 + .../3-workload/rbd_notification_tests.yaml | 21 + .../rbd/supported/centos_7.4.yaml | 1 + .../rbd/supported/rhel_7.5.yaml | 1 + .../rbd/supported/ubuntu_16.04.yaml | 1 + .../rbd/supported/ubuntu_18.04.yaml | 1 + .../parallel/1-jewel-install/jewel.yaml | 1 - .../kraken-x/parallel/0-cluster/start.yaml | 2 +- ceph/qa/suites/upgrade/luminous-p2p/% | 0 .../point-to-point-upgrade.yaml | 120 +-- ceph/qa/suites/upgrade/luminous-p2p/supported | 1 + .../luminous-x/parallel/0-cluster/start.yaml | 2 +- .../distros/centos_latest.yaml | 1 - .../distros/ubuntu_latest.yaml | 1 - ceph/qa/tasks/ceph.py | 2 + ceph/qa/tasks/ceph_deploy.py | 13 + ceph/qa/tasks/ceph_manager.py | 2 +- ceph/qa/tasks/cephfs/filesystem.py | 16 +- ceph/qa/tasks/cephfs/fuse_mount.py | 12 +- ceph/qa/tasks/cephfs/kernel_mount.py | 2 +- ceph/qa/tasks/cephfs/test_client_recovery.py | 41 ++ ceph/qa/tasks/cephfs/test_exports.py | 40 + ceph/qa/tasks/cephfs/test_strays.py | 70 -- ceph/qa/tasks/cephfs/test_volume_client.py | 118 ++- ceph/qa/tasks/filestore_idempotent.py | 6 +- ceph/qa/tasks/osd_max_pg_per_osd.py | 10 +- ceph/qa/tasks/radosgw_admin_rest.py | 1 + .../ceph-tests/ceph-admin-commands.sh | 1 - ceph/qa/workunits/fs/misc/rstats.sh | 80 ++ ceph/qa/workunits/mon/crush_ops.sh | 11 + .../rados/test_envlibrados_for_rocksdb.sh | 2 +- .../rados/test_large_omap_detection.py | 130 ++++ ceph/qa/workunits/rados/test_pool_access.sh | 99 ++- ceph/qa/workunits/rbd/import_export.sh | 14 +- ceph/qa/workunits/rbd/permissions.sh | 92 +++ ceph/qa/workunits/rbd/rbd_mirror.sh | 2 +- ceph/qa/workunits/rest/test_mgr_rest_api.py | 2 + ceph/selinux/ceph.te | 3 + ceph/src/.git_version | 4 +- ceph/src/CMakeLists.txt | 10 +- ceph/src/arch/arm.c | 43 +- ceph/src/arch/arm.h | 1 + ceph/src/auth/Auth.h | 5 + ceph/src/auth/AuthAuthorizeHandler.h | 4 +- ceph/src/auth/cephx/CephxAuthorizeHandler.cc | 12 +- ceph/src/auth/cephx/CephxAuthorizeHandler.h | 3 +- ceph/src/auth/cephx/CephxProtocol.cc | 59 +- ceph/src/auth/cephx/CephxProtocol.h | 37 +- ceph/src/auth/cephx/CephxServiceHandler.cc | 4 +- ceph/src/auth/cephx/CephxSessionHandler.cc | 94 ++- .../src/auth/none/AuthNoneAuthorizeHandler.cc | 11 +- ceph/src/auth/none/AuthNoneAuthorizeHandler.h | 3 +- ceph/src/auth/none/AuthNoneProtocol.h | 3 + .../unknown/AuthUnknownAuthorizeHandler.cc | 11 +- .../unknown/AuthUnknownAuthorizeHandler.h | 3 +- ceph/src/ceph-disk/ceph_disk/main.py | 7 +- ceph/src/ceph-volume/ceph_volume/__init__.py | 11 + ceph/src/ceph-volume/ceph_volume/api/lvm.py | 2 +- .../ceph_volume/devices/lvm/activate.py | 2 + .../ceph_volume/devices/lvm/create.py | 2 +- .../ceph_volume/devices/lvm/listing.py | 40 +- .../ceph_volume/devices/lvm/prepare.py | 2 +- .../ceph-volume/ceph_volume/tests/conftest.py | 18 +- .../tests/devices/lvm/test_listing.py | 85 ++- .../centos7/bluestore/create/group_vars/all | 2 +- .../centos7/bluestore/dmcrypt/group_vars/all | 2 +- .../lvm/centos7/bluestore/dmcrypt/test.yml | 38 +- .../centos7/filestore/create/group_vars/all | 2 +- .../centos7/filestore/dmcrypt/group_vars/all | 2 +- .../lvm/centos7/filestore/dmcrypt/test.yml | 28 +- .../lvm/playbooks/test_bluestore.yml | 28 +- .../lvm/playbooks/test_filestore.yml | 30 +- .../ceph_volume/tests/functional/lvm/tox.ini | 4 + .../xenial/bluestore/create/group_vars/all | 2 +- .../xenial/bluestore/dmcrypt/group_vars/all | 2 +- .../lvm/xenial/bluestore/dmcrypt/test.yml | 48 +- .../xenial/filestore/create/group_vars/all | 2 +- .../xenial/filestore/dmcrypt/group_vars/all | 2 +- .../lvm/xenial/filestore/dmcrypt/test.yml | 28 +- .../centos7/bluestore/activate/group_vars/all | 2 +- .../bluestore/dmcrypt-luks/group_vars/all | 2 +- .../bluestore/dmcrypt-plain/group_vars/all | 2 +- .../centos7/filestore/activate/group_vars/all | 2 +- .../filestore/dmcrypt-luks/group_vars/all | 2 +- .../filestore/dmcrypt-plain/group_vars/all | 2 +- .../tests/functional/simple/tox.ini | 4 + .../xenial/bluestore/activate/group_vars/all | 2 +- .../bluestore/dmcrypt-luks/group_vars/all | 2 +- .../bluestore/dmcrypt-plain/group_vars/all | 2 +- .../xenial/filestore/activate/group_vars/all | 2 +- .../filestore/dmcrypt-luks/group_vars/all | 2 +- .../filestore/dmcrypt-plain/group_vars/all | 2 +- .../ceph_volume/tests/util/test_prepare.py | 28 + .../ceph-volume/ceph_volume/util/prepare.py | 4 +- .../ceph-volume/ceph_volume/util/system.py | 1 + ceph/src/ceph_fuse.cc | 19 +- ceph/src/client/Client.cc | 315 ++++---- ceph/src/client/Client.h | 36 +- ceph/src/client/Inode.cc | 33 +- ceph/src/client/Inode.h | 6 +- ceph/src/client/MetaRequest.h | 2 +- ceph/src/client/UserPerm.h | 4 +- ceph/src/client/fuse_ll.cc | 73 +- ceph/src/cls/rgw/cls_rgw.cc | 25 +- ceph/src/common/DecayCounter.cc | 30 +- ceph/src/common/DecayCounter.h | 6 +- ceph/src/common/Preforker.h | 7 +- ceph/src/common/bounded_key_counter.h | 6 +- ceph/src/common/code_environment.cc | 4 + ceph/src/common/compat.cc | 78 ++ ceph/src/common/crc32c_aarch64.c | 7 +- ceph/src/common/legacy_config_opts.h | 16 +- ceph/src/common/obj_bencher.cc | 2 +- ceph/src/common/options.cc | 232 ++++-- ceph/src/crush/CrushCompiler.cc | 12 +- ceph/src/crush/CrushTreeDumper.h | 4 +- ceph/src/crush/CrushWrapper.cc | 229 ++++-- ceph/src/crush/CrushWrapper.h | 29 +- ceph/src/crush/builder.c | 2 +- ceph/src/crush/crush.h | 4 +- ceph/src/crush/mapper.c | 4 +- ceph/src/global/global_init.cc | 20 +- ceph/src/include/ceph_features.h | 4 +- ceph/src/include/ceph_fs.h | 1 + ceph/src/include/cephfs/libcephfs.h | 2 + ceph/src/include/compat.h | 3 + ceph/src/include/interval_set.h | 21 +- ceph/src/include/msgr.h | 2 +- ceph/src/java/CMakeLists.txt | 44 +- ceph/src/libcephfs.cc | 6 + ceph/src/librados/IoCtxImpl.cc | 9 +- ceph/src/librbd/ImageCtx.cc | 4 + ceph/src/librbd/ImageCtx.h | 2 + ceph/src/librbd/ImageWatcher.cc | 2 +- ceph/src/librbd/Watcher.cc | 142 ++-- ceph/src/librbd/Watcher.h | 18 +- ceph/src/librbd/journal/Replay.cc | 34 +- ceph/src/librbd/journal/Replay.h | 13 +- ceph/src/log/Log.cc | 15 +- ceph/src/mds/Beacon.cc | 1 + ceph/src/mds/CDir.cc | 130 ++-- ceph/src/mds/CDir.h | 6 +- ceph/src/mds/CInode.cc | 12 +- ceph/src/mds/CInode.h | 3 +- ceph/src/mds/Capability.h | 2 +- ceph/src/mds/FSMap.cc | 1 + ceph/src/mds/FSMap.h | 1 + ceph/src/mds/Locker.cc | 8 +- ceph/src/mds/MDBalancer.cc | 478 ++++++++---- ceph/src/mds/MDBalancer.h | 28 +- ceph/src/mds/MDCache.cc | 295 +++++--- ceph/src/mds/MDCache.h | 22 +- ceph/src/mds/MDSCacheObject.h | 2 +- ceph/src/mds/MDSDaemon.cc | 15 +- ceph/src/mds/MDSDaemon.h | 3 +- ceph/src/mds/MDSRank.cc | 87 ++- ceph/src/mds/MDSRank.h | 10 +- ceph/src/mds/Migrator.cc | 149 ++-- ceph/src/mds/Migrator.h | 23 +- ceph/src/mds/Mutation.h | 3 +- ceph/src/mds/PurgeQueue.cc | 2 +- ceph/src/mds/Server.cc | 185 +++-- ceph/src/mds/Server.h | 5 +- ceph/src/mds/SessionMap.h | 17 +- ceph/src/mds/journal.cc | 12 +- ceph/src/mds/mdstypes.cc | 10 + ceph/src/mds/mdstypes.h | 15 +- ceph/src/messages/MClientCaps.h | 72 +- ceph/src/messages/MHeartbeat.h | 2 +- ceph/src/messages/MMDSMap.h | 2 +- ceph/src/messages/MOSDMap.h | 26 +- ceph/src/messages/MOSDRepScrub.h | 46 +- ceph/src/messages/MOSDRepScrubMap.h | 10 +- ceph/src/mgr/ActivePyModules.cc | 28 +- ceph/src/mgr/DaemonServer.cc | 20 +- ceph/src/mgr/DaemonServer.h | 16 +- ceph/src/mgr/DaemonState.cc | 12 +- ceph/src/mgr/DaemonState.h | 28 +- ceph/src/mon/AuthMonitor.cc | 27 +- ceph/src/mon/CMakeLists.txt | 3 +- ceph/src/mon/ConfigKeyService.cc | 20 +- ceph/src/mon/LogMonitor.cc | 3 +- ceph/src/mon/MDSMonitor.cc | 485 +++++++------ ceph/src/mon/MDSMonitor.h | 20 +- ceph/src/mon/MgrMonitor.cc | 2 +- ceph/src/mon/MonCommands.h | 2 +- ceph/src/mon/Monitor.cc | 5 +- ceph/src/mon/Monitor.h | 3 +- ceph/src/mon/OSDMonitor.cc | 380 +++++++--- ceph/src/mon/OSDMonitor.h | 26 +- ceph/src/mon/PGMap.cc | 62 +- ceph/src/mon/Paxos.h | 8 - ceph/src/mon/PaxosFSMap.h | 63 ++ ceph/src/msg/Dispatcher.h | 7 +- ceph/src/msg/Messenger.h | 6 +- ceph/src/msg/async/AsyncConnection.cc | 54 +- ceph/src/msg/async/AsyncConnection.h | 1 + ceph/src/msg/async/AsyncMessenger.h | 5 +- ceph/src/msg/simple/Pipe.cc | 50 +- ceph/src/msg/simple/SimpleMessenger.cc | 7 +- ceph/src/msg/simple/SimpleMessenger.h | 6 +- ceph/src/os/ObjectStore.h | 3 + ceph/src/os/bluestore/BlueStore.cc | 90 +-- ceph/src/os/bluestore/BlueStore.h | 25 +- ceph/src/os/bluestore/KernelDevice.cc | 21 +- ceph/src/os/bluestore/StupidAllocator.cc | 4 +- ceph/src/os/bluestore/aio.cc | 5 +- ceph/src/os/bluestore/bluefs_types.h | 2 +- ceph/src/os/bluestore/bluestore_types.cc | 5 + ceph/src/os/bluestore/bluestore_types.h | 2 + ceph/src/os/filestore/FileJournal.cc | 21 +- ceph/src/os/filestore/FileStore.cc | 21 +- ceph/src/os/filestore/FileStore.h | 4 + ceph/src/osd/ECBackend.cc | 175 +++-- ceph/src/osd/ECBackend.h | 19 +- ceph/src/osd/OSD.cc | 52 +- ceph/src/osd/OSD.h | 6 +- ceph/src/osd/OSDMap.cc | 200 ++--- ceph/src/osd/OSDMap.h | 21 + ceph/src/osd/PG.cc | 443 ++++++++---- ceph/src/osd/PG.h | 74 +- ceph/src/osd/PGBackend.cc | 158 ++-- ceph/src/osd/PGBackend.h | 27 +- ceph/src/osd/PGTransaction.h | 8 +- ceph/src/osd/PrimaryLogPG.cc | 140 +++- ceph/src/osd/PrimaryLogPG.h | 12 +- ceph/src/osd/ReplicatedBackend.cc | 189 +++-- ceph/src/osd/ReplicatedBackend.h | 10 +- ceph/src/osd/SnapMapper.cc | 5 +- ceph/src/osd/osd_types.cc | 37 +- ceph/src/osd/osd_types.h | 74 +- ceph/src/osdc/Journaler.cc | 18 + ceph/src/osdc/Journaler.h | 3 + ceph/src/osdc/ObjectCacher.cc | 92 ++- ceph/src/osdc/ObjectCacher.h | 8 +- ceph/src/osdc/Objecter.cc | 23 +- ceph/src/pybind/ceph_volume_client.py | 121 +++- ceph/src/pybind/mgr/influx/module.py | 213 ++++-- ceph/src/pybind/mgr/mgr_module.py | 38 +- ceph/src/pybind/mgr/prometheus/module.py | 81 ++- ceph/src/pybind/mgr/restful/common.py | 3 +- ceph/src/pybind/mgr/restful/module.py | 9 +- ceph/src/pybind/rados/rados.pyx | 106 +++ ceph/src/pybind/rbd/rbd.pyx | 32 +- ceph/src/rgw/rgw_acl_swift.cc | 12 +- ceph/src/rgw/rgw_acl_swift.h | 4 +- ceph/src/rgw/rgw_admin.cc | 335 ++++++++- ceph/src/rgw/rgw_asio_frontend.cc | 135 ++-- ceph/src/rgw/rgw_auth.cc | 68 +- ceph/src/rgw/rgw_auth.h | 47 +- ceph/src/rgw/rgw_auth_registry.h | 18 +- ceph/src/rgw/rgw_auth_s3.cc | 4 +- ceph/src/rgw/rgw_auth_s3.h | 9 +- ceph/src/rgw/rgw_cache.cc | 14 +- ceph/src/rgw/rgw_cache.h | 2 +- ceph/src/rgw/rgw_civetweb.cc | 2 +- ceph/src/rgw/rgw_civetweb_frontend.cc | 23 +- ceph/src/rgw/rgw_common.h | 2 +- ceph/src/rgw/rgw_coroutine.cc | 4 +- ceph/src/rgw/rgw_cr_rados.cc | 15 +- ceph/src/rgw/rgw_cr_rados.h | 10 +- ceph/src/rgw/rgw_data_sync.cc | 289 ++++++-- ceph/src/rgw/rgw_data_sync.h | 50 +- ceph/src/rgw/rgw_file.cc | 38 +- ceph/src/rgw/rgw_file.h | 43 ++ ceph/src/rgw/rgw_frontend.cc | 8 +- ceph/src/rgw/rgw_frontend.h | 19 +- ceph/src/rgw/rgw_http_client.cc | 9 +- ceph/src/rgw/rgw_iam_policy.cc | 6 +- ceph/src/rgw/rgw_iam_policy.h | 4 +- ceph/src/rgw/rgw_lc_s3.cc | 2 + ceph/src/rgw/rgw_main.cc | 9 +- ceph/src/rgw/rgw_op.cc | 79 +- ceph/src/rgw/rgw_op.h | 21 + ceph/src/rgw/rgw_period_puller.cc | 8 +- ceph/src/rgw/rgw_rados.cc | 110 +-- ceph/src/rgw/rgw_rados.h | 9 +- ceph/src/rgw/rgw_realm_reloader.cc | 4 +- ceph/src/rgw/rgw_rest.cc | 15 +- ceph/src/rgw/rgw_rest_log.cc | 10 +- ceph/src/rgw/rgw_rest_s3.cc | 1 + ceph/src/rgw/rgw_rest_s3.h | 12 +- ceph/src/rgw/rgw_rest_swift.cc | 14 +- ceph/src/rgw/rgw_rest_user.cc | 9 +- ceph/src/rgw/rgw_swift_auth.h | 6 +- ceph/src/rgw/rgw_sync.cc | 18 +- ceph/src/rgw/rgw_sync_module_es.cc | 5 + ceph/src/rgw/rgw_torrent.cc | 28 +- ceph/src/rgw/rgw_user.cc | 2 +- ceph/src/script/build-integration-branch | 65 ++ ceph/src/test/CMakeLists.txt | 2 +- ceph/src/test/cli/crushtool/choose-args.crush | 10 +- ceph/src/test/cli/crushtool/choose-args.t | 14 +- ceph/src/test/cli/radosgw-admin/help.t | 19 +- ceph/src/test/crush/CrushWrapper.cc | 4 +- ceph/src/test/fs/CMakeLists.txt | 20 +- ceph/src/test/fs/test_trim_caps.cc | 83 +++ ceph/src/test/libcephfs/test.cc | 62 ++ ceph/src/test/librados/aio.cc | 3 + ceph/src/test/librados/list.cc | 8 +- ceph/src/test/librados/test_common.cc | 81 ++- ceph/src/test/librados/test_common.h | 2 + .../test/librbd/journal/test_mock_Replay.cc | 54 ++ ceph/src/test/librbd/mock/MockImageCtx.h | 2 + ceph/src/test/messenger/simple_dispatcher.h | 7 +- ceph/src/test/messenger/xio_dispatcher.h | 3 +- ceph/src/test/msgr/perf_msgr_client.cc | 3 +- ceph/src/test/msgr/perf_msgr_server.cc | 3 +- ceph/src/test/msgr/test_msgr.cc | 9 +- .../objectstore/DeterministicOpSequence.cc | 211 +++--- .../objectstore/DeterministicOpSequence.h | 35 +- ceph/src/test/objectstore/FileStoreDiff.cc | 110 +-- .../test/objectstore/TestObjectStoreState.h | 2 +- ceph/src/test/objectstore/run_seed_to.sh | 11 +- .../src/test/objectstore/run_seed_to_range.sh | 5 +- ceph/src/test/pybind/test_rados.py | 37 + .../test_mock_BootstrapRequest.cc | 99 ++- .../test_mock_PrepareLocalImageRequest.cc | 48 ++ .../src/test/rbd_mirror/test_ImageReplayer.cc | 9 +- .../rbd_mirror/test_mock_ImageReplayer.cc | 33 +- ceph/src/test/rgw/rgw_multi/tests.py | 17 +- ceph/src/test/rgw/test_rgw_iam_policy.cc | 6 +- ceph/src/tools/cephfs/Resetter.cc | 21 +- ceph/src/tools/rbd/Utils.h | 3 + ceph/src/tools/rbd/action/Export.cc | 58 +- ceph/src/tools/rbd/action/Import.cc | 107 ++- ceph/src/tools/rbd_mirror/ImageReplayer.cc | 76 +- ceph/src/tools/rbd_mirror/ImageReplayer.h | 4 +- ceph/src/tools/rbd_mirror/ImageSync.cc | 20 +- ceph/src/tools/rbd_mirror/InstanceWatcher.cc | 19 +- ceph/src/tools/rbd_mirror/InstanceWatcher.h | 1 + ceph/src/tools/rbd_mirror/PoolReplayer.cc | 9 +- .../image_replayer/BootstrapRequest.cc | 106 +-- .../image_replayer/BootstrapRequest.h | 4 + .../image_replayer/IsPrimaryRequest.cc | 4 +- .../image_replayer/OpenLocalImageRequest.cc | 6 +- .../PrepareLocalImageRequest.cc | 36 + .../image_replayer/PrepareLocalImageRequest.h | 16 +- ceph/src/tools/rbd_nbd/rbd-nbd.cc | 6 +- 450 files changed, 9936 insertions(+), 3812 deletions(-) create mode 100644 ceph/doc/dev/testing.rst create mode 100644 ceph/doc/mgr/balancer.rst create mode 100644 ceph/qa/distros/all/rhel_7.5.yaml create mode 100644 ceph/qa/distros/all/ubuntu_18.04.yaml create mode 100644 ceph/qa/overrides/more-active-recovery.yaml create mode 100755 ceph/qa/standalone/osd/ec-error-rollforward.sh rename ceph/qa/suites/{upgrade/luminous-x/point-to-point-x => fs/bugs/client_trim_caps}/% (100%) create mode 120000 ceph/qa/suites/fs/bugs/client_trim_caps/begin.yaml create mode 100644 ceph/qa/suites/fs/bugs/client_trim_caps/clusters/small-cluster.yaml create mode 120000 ceph/qa/suites/fs/bugs/client_trim_caps/objectstore/bluestore.yaml create mode 100644 ceph/qa/suites/fs/bugs/client_trim_caps/overrides/+ create mode 120000 ceph/qa/suites/fs/bugs/client_trim_caps/overrides/debug.yaml create mode 120000 ceph/qa/suites/fs/bugs/client_trim_caps/overrides/frag_enable.yaml create mode 120000 ceph/qa/suites/fs/bugs/client_trim_caps/overrides/no_client_pidfile.yaml create mode 120000 ceph/qa/suites/fs/bugs/client_trim_caps/overrides/whitelist_health.yaml create mode 120000 ceph/qa/suites/fs/bugs/client_trim_caps/overrides/whitelist_wrongly_marked_down.yaml create mode 100644 ceph/qa/suites/fs/bugs/client_trim_caps/tasks/trim-i22073.yaml create mode 100644 ceph/qa/suites/rados/singleton-nomsgr/all/large-omap-object-warnings.yaml create mode 120000 ceph/qa/suites/rados/thrash-erasure-code-big/recovery-overrides create mode 120000 ceph/qa/suites/rados/thrash-erasure-code-isa/recovery-overrides create mode 120000 ceph/qa/suites/rados/thrash-erasure-code-overwrites/recovery-overrides create mode 120000 ceph/qa/suites/rados/thrash-erasure-code-shec/recovery-overrides create mode 120000 ceph/qa/suites/rados/thrash-erasure-code/recovery-overrides create mode 100644 ceph/qa/suites/rados/thrash/2-recovery-overrides/$ create mode 100644 ceph/qa/suites/rados/thrash/2-recovery-overrides/default.yaml create mode 120000 ceph/qa/suites/rados/thrash/2-recovery-overrides/more-active-recovery.yaml delete mode 100644 ceph/qa/suites/rbd/basic/msgr-failures/many.yaml create mode 100644 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/% create mode 100644 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/0-cluster/+ create mode 100644 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/0-cluster/openstack.yaml create mode 100644 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/0-cluster/start.yaml create mode 100644 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/1-install/luminous-client-x.yaml create mode 100644 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/2-workload/rbd_api_tests.yaml create mode 100644 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/2-workload/rbd_cli_import_export.yaml create mode 120000 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/supported/centos_7.4.yaml create mode 120000 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/supported/rhel_7.5.yaml create mode 120000 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/supported/ubuntu_16.04.yaml create mode 120000 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/supported/ubuntu_18.04.yaml create mode 100644 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/% create mode 100644 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/0-cluster/+ create mode 100644 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/0-cluster/openstack.yaml create mode 100644 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/0-cluster/start.yaml create mode 100644 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/1-install/luminous-client-x.yaml create mode 100644 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/2-features/defaults.yaml create mode 100644 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/2-features/layering.yaml create mode 100644 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/3-workload/rbd_notification_tests.yaml create mode 120000 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/supported/centos_7.4.yaml create mode 120000 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/supported/rhel_7.5.yaml create mode 120000 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/supported/ubuntu_16.04.yaml create mode 120000 ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/supported/ubuntu_18.04.yaml create mode 100644 ceph/qa/suites/upgrade/luminous-p2p/% rename ceph/qa/suites/upgrade/{luminous-x/point-to-point-x => luminous-p2p}/point-to-point-upgrade.yaml (62%) create mode 120000 ceph/qa/suites/upgrade/luminous-p2p/supported delete mode 120000 ceph/qa/suites/upgrade/luminous-x/point-to-point-x/distros/centos_latest.yaml delete mode 120000 ceph/qa/suites/upgrade/luminous-x/point-to-point-x/distros/ubuntu_latest.yaml create mode 100755 ceph/qa/workunits/fs/misc/rstats.sh create mode 100755 ceph/qa/workunits/rados/test_large_omap_detection.py create mode 100644 ceph/src/common/compat.cc create mode 100644 ceph/src/mon/PaxosFSMap.h create mode 100755 ceph/src/script/build-integration-branch create mode 100644 ceph/src/test/fs/test_trim_caps.cc diff --git a/Makefile b/Makefile index 3c2451983..9bc9fbe46 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ -RELEASE=5.1 +RELEASE=5.2 PACKAGE=ceph -VER=12.2.5 +VER=12.2.7 DEBREL=pve1 SRCDIR=ceph diff --git a/ceph/CMakeLists.txt b/ceph/CMakeLists.txt index aa90ba65d..2e2df1f6e 100644 --- a/ceph/CMakeLists.txt +++ b/ceph/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 2.8.11) project(ceph) -set(VERSION 12.2.5) +set(VERSION 12.2.7) if(POLICY CMP0046) # Tweak policies (this one disables "missing" dependency warning) diff --git a/ceph/alpine/APKBUILD b/ceph/alpine/APKBUILD index d280aacfb..90553c49e 100644 --- a/ceph/alpine/APKBUILD +++ b/ceph/alpine/APKBUILD @@ -1,7 +1,7 @@ # Contributor: John Coyle # Maintainer: John Coyle pkgname=ceph -pkgver=12.2.5 +pkgver=12.2.7 pkgrel=0 pkgdesc="Ceph is a distributed object store and file system" pkgusers="ceph" @@ -63,7 +63,7 @@ makedepends=" xmlstarlet yasm " -source="ceph-12.2.5.tar.bz2" +source="ceph-12.2.7.tar.bz2" subpackages=" $pkgname-base $pkgname-common @@ -116,7 +116,7 @@ _sysconfdir=/etc _udevrulesdir=/etc/udev/rules.d _python_sitelib=/usr/lib/python2.7/site-packages -builddir=$srcdir/ceph-12.2.5 +builddir=$srcdir/ceph-12.2.7 build() { export CEPH_BUILD_VIRTUALENV=$builddir diff --git a/ceph/ceph.spec b/ceph/ceph.spec index bbb86d452..08ccdb6c2 100644 --- a/ceph/ceph.spec +++ b/ceph/ceph.spec @@ -61,7 +61,7 @@ # main package definition ################################################################################# Name: ceph -Version: 12.2.5 +Version: 12.2.7 Release: 0%{?dist} %if 0%{?fedora} || 0%{?rhel} Epoch: 2 @@ -77,7 +77,7 @@ License: LGPL-2.1 and CC-BY-SA-3.0 and GPL-2.0 and BSL-1.0 and BSD-3-Clause and Group: System/Filesystems %endif URL: http://ceph.com/ -Source0: http://ceph.com/download/ceph-12.2.5.tar.bz2 +Source0: http://ceph.com/download/ceph-12.2.7.tar.bz2 %if 0%{?suse_version} %if 0%{?is_opensuse} ExclusiveArch: x86_64 aarch64 ppc64 ppc64le @@ -779,7 +779,7 @@ python-rbd, python-rgw or python-cephfs instead. # common ################################################################################# %prep -%autosetup -p1 -n ceph-12.2.5 +%autosetup -p1 -n ceph-12.2.7 %build %if 0%{with cephfs_java} @@ -1447,6 +1447,8 @@ fi %else /usr/lib/systemd/systemd-sysctl %{_sysctldir}/90-ceph-osd.conf > /dev/null 2>&1 || : %endif +# work around https://tracker.ceph.com/issues/24903 +chown -h ceph:ceph /var/lib/ceph/osd/*/block* 2>&1 > /dev/null || : %preun osd %if 0%{?suse_version} diff --git a/ceph/ceph.spec.in b/ceph/ceph.spec.in index 2f89a9bc1..05e3e330e 100644 --- a/ceph/ceph.spec.in +++ b/ceph/ceph.spec.in @@ -1447,6 +1447,8 @@ fi %else /usr/lib/systemd/systemd-sysctl %{_sysctldir}/90-ceph-osd.conf > /dev/null 2>&1 || : %endif +# work around https://tracker.ceph.com/issues/24903 +chown -h ceph:ceph /var/lib/ceph/osd/*/block* 2>&1 > /dev/null || : %preun osd %if 0%{?suse_version} diff --git a/ceph/debian/ceph-osd.postinst b/ceph/debian/ceph-osd.postinst index 5e44548fe..8acfc0743 100644 --- a/ceph/debian/ceph-osd.postinst +++ b/ceph/debian/ceph-osd.postinst @@ -25,6 +25,8 @@ case "$1" in configure) [ -x /etc/init.d/procps ] && invoke-rc.d procps restart || : [ -x /sbin/start ] && start ceph-osd-all || : + # work around https://tracker.ceph.com/issues/24903 + chown -h ceph:ceph /var/lib/ceph/osd/*/block* 2>&1 > /dev/null || : ;; abort-upgrade|abort-remove|abort-deconfigure) : diff --git a/ceph/debian/changelog b/ceph/debian/changelog index c3a8efa2d..a95f9dc26 100644 --- a/ceph/debian/changelog +++ b/ceph/debian/changelog @@ -1,3 +1,15 @@ +ceph (12.2.7-1) stable; urgency=medium + + * New upstream release + + -- Ceph Release Team Mon, 16 Jul 2018 16:00:29 +0000 + +ceph (12.2.6-1) stable; urgency=medium + + * New upstream release + + -- Ceph Release Team Mon, 09 Jul 2018 16:18:46 +0000 + ceph (12.2.5-1) stable; urgency=medium * New upstream release diff --git a/ceph/doc/_templates/smarttoc.html b/ceph/doc/_templates/smarttoc.html index 8ec60cdf4..7a3fd3f2b 100644 --- a/ceph/doc/_templates/smarttoc.html +++ b/ceph/doc/_templates/smarttoc.html @@ -10,7 +10,7 @@ #}

{{ _('Table Of Contents') }}

-{{ toctree(maxdepth=-1) }} +{{ toctree(maxdepth=-1, includehidden=True) }} diff --git a/ceph/doc/ceph-volume/index.rst b/ceph/doc/ceph-volume/index.rst index 2ec462ebe..387596d61 100644 --- a/ceph/doc/ceph-volume/index.rst +++ b/ceph/doc/ceph-volume/index.rst @@ -22,7 +22,13 @@ Migrating --------- Starting on Ceph version 12.2.2, ``ceph-disk`` is deprecated. Deprecation warnings will show up that will link to this page. It is strongly suggested -that users start consuming ``ceph-volume``. +that users start consuming ``ceph-volume``. There are two paths for migrating: + +#. Keep OSDs deployed with ``ceph-disk``: The :ref:`ceph-volume-simple` command + provides a way to take over the management while disabling ``ceph-disk`` + triggers. +#. Redeploy existing OSDs with ``ceph-volume``: This is covered in depth on + :ref:`rados-replacing-an-osd` New deployments ^^^^^^^^^^^^^^^ diff --git a/ceph/doc/ceph-volume/lvm/activate.rst b/ceph/doc/ceph-volume/lvm/activate.rst index 66bed5a82..e757f5266 100644 --- a/ceph/doc/ceph-volume/lvm/activate.rst +++ b/ceph/doc/ceph-volume/lvm/activate.rst @@ -22,6 +22,18 @@ need to be supplied. For example:: .. note:: The UUID is stored in the ``fsid`` file in the OSD path, which is generated when :ref:`ceph-volume-lvm-prepare` is used. +Activating all OSDs +------------------- +It is possible to activate all existing OSDs at once by using the ``--all`` +flag. For example:: + + ceph-volume lvm activate --all + +This call will inspect all the OSDs created by ceph-volume that are inactive +and will activate them one by one. If any of the OSDs are already running, it +will report them in the command output and skip them, making it safe to rerun +(idempotent). + requiring uuids ^^^^^^^^^^^^^^^ The :term:`OSD uuid` is being required as an extra step to ensure that the @@ -29,6 +41,13 @@ right OSD is being activated. It is entirely possible that a previous OSD with the same id exists and would end up activating the incorrect one. +dmcrypt +^^^^^^^ +If the OSD was prepared with dmcrypt by ceph-volume, there is no need to +specify ``--dmcrypt`` on the command line again (that flag is not available for +the ``activate`` subcommand). An encrypted OSD will be automatically detected. + + Discovery --------- With either existing OSDs or new ones being activated, a *discovery* process is @@ -63,9 +82,11 @@ The systemd unit will look for the matching OSD device, and by looking at its Existing OSDs ------------- -For exsiting OSDs that have been deployed with different tooling, the only way -to port them over to the new mechanism is to prepare them again (losing data). -See :ref:`ceph-volume-lvm-existing-osds` for details on how to proceed. +For existing OSDs that have been deployed with ``ceph-disk``, they need to be +scanned and activated :ref:`using the simple sub-command `. +If a different tooling was used then the only way to port them over to the new +mechanism is to prepare them again (losing data). See +:ref:`ceph-volume-lvm-existing-osds` for details on how to proceed. Summary ------- diff --git a/ceph/doc/ceph-volume/lvm/encryption.rst b/ceph/doc/ceph-volume/lvm/encryption.rst index 02a447b57..6bdec7c44 100644 --- a/ceph/doc/ceph-volume/lvm/encryption.rst +++ b/ceph/doc/ceph-volume/lvm/encryption.rst @@ -3,10 +3,11 @@ Encryption ========== -Logical volumes can be encrypted using ``dmcrypt``. Encryption can be done in -different ways, specially with LVM. ``ceph-volume`` is somewhat opinionated -with the way it sets up encryption with logical volumes so that the process is -consistent and robust. +Logical volumes can be encrypted using ``dmcrypt`` by specifying the +``--dmcrypt`` flag when creating OSDs. Encryption can be done in different ways, +specially with LVM. ``ceph-volume`` is somewhat opinionated with the way it +sets up encryption with logical volumes so that the process is consistent and +robust. In this case, ``ceph-volume lvm`` follows these constraints: diff --git a/ceph/doc/ceph-volume/lvm/list.rst b/ceph/doc/ceph-volume/lvm/list.rst index 19e06000b..718154b10 100644 --- a/ceph/doc/ceph-volume/lvm/list.rst +++ b/ceph/doc/ceph-volume/lvm/list.rst @@ -39,6 +39,7 @@ one with a physical device may look similar to:: data uuid SlEgHe-jX1H-QBQk-Sce0-RUls-8KlY-g8HgcZ journal device /dev/journals/journal1 data device /dev/test_group/data-lv2 + devices /dev/sda [data] /dev/test_group/data-lv2 @@ -50,6 +51,7 @@ one with a physical device may look similar to:: data uuid SlEgHe-jX1H-QBQk-Sce0-RUls-8KlY-g8HgcZ journal device /dev/journals/journal1 data device /dev/test_group/data-lv2 + devices /dev/sdb ====== osd.0 ======= @@ -63,11 +65,18 @@ one with a physical device may look similar to:: data uuid TUpfel-Q5ZT-eFph-bdGW-SiNW-l0ag-f5kh00 journal device /dev/sdd1 data device /dev/test_group/data-lv1 + devices /dev/sdc [journal] /dev/sdd1 PARTUUID cd72bd28-002a-48da-bdf6-d5b993e84f3f + +For logical volumes the ``devices`` key is populated with the physical devices +associated with the logical volume. Since LVM allows multiple physical devices +to be part of a logical volume, the value will be comma separated when using +``pretty``, but an array when using ``json``. + .. note:: Tags are displayed in a readable format. The ``osd id`` key is stored as a ``ceph.osd_id`` tag. For more information on lvm tag conventions see :ref:`ceph-volume-lvm-tag-api` @@ -96,6 +105,7 @@ can be listed in the following way:: data uuid SlEgHe-jX1H-QBQk-Sce0-RUls-8KlY-g8HgcZ journal device /dev/journals/journal1 data device /dev/test_group/data-lv2 + devices /dev/sdc .. note:: Tags are displayed in a readable format. The ``osd id`` key is stored @@ -134,6 +144,7 @@ output (note how tags aren't modified):: { "0": [ { + "devices": ["/dev/sda"], "lv_name": "data-lv1", "lv_path": "/dev/test_group/data-lv1", "lv_tags": "ceph.cluster_fsid=ce454d91-d748-4751-a318-ff7f7aa18ffd,ceph.data_device=/dev/test_group/data-lv1,ceph.data_uuid=TUpfel-Q5ZT-eFph-bdGW-SiNW-l0ag-f5kh00,ceph.journal_device=/dev/sdd1,ceph.journal_uuid=cd72bd28-002a-48da-bdf6-d5b993e84f3f,ceph.osd_fsid=943949f0-ce37-47ca-a33c-3413d46ee9ec,ceph.osd_id=0,ceph.type=data", diff --git a/ceph/doc/ceph-volume/lvm/prepare.rst b/ceph/doc/ceph-volume/lvm/prepare.rst index de29a782a..700d1216a 100644 --- a/ceph/doc/ceph-volume/lvm/prepare.rst +++ b/ceph/doc/ceph-volume/lvm/prepare.rst @@ -42,6 +42,10 @@ The API call looks like:: ceph-volume prepare --filestore --data data --journal journal +For enabling :ref:`encryption `, the ``--dmcrypt`` flag is required:: + + ceph-volume lvm prepare --filestore --dmcrypt --data volume_group/lv_name --journal journal + There is flexibility to use a raw device or partition as well for ``--data`` that will be converted to a logical volume. This is not ideal in all situations since ``ceph-volume`` is just going to create a unique volume group and @@ -148,6 +152,9 @@ A raw device can be specified in the same way:: ceph-volume lvm prepare --bluestore --data /path/to/device +For enabling :ref:`encryption `, the ``--dmcrypt`` flag is required:: + + ceph-volume lvm prepare --bluestore --dmcrypt --data vg/lv If a ``block.db`` or a ``block.wal`` is needed (they are optional for bluestore) they can be specified with ``--block.db`` and ``--block.wal`` diff --git a/ceph/doc/ceph-volume/simple/index.rst b/ceph/doc/ceph-volume/simple/index.rst index 6f2534a73..315dea99a 100644 --- a/ceph/doc/ceph-volume/simple/index.rst +++ b/ceph/doc/ceph-volume/simple/index.rst @@ -17,3 +17,16 @@ Implements the functionality needed to manage OSDs from the ``simple`` subcomman By *taking over* management, it disables all ``ceph-disk`` systemd units used to trigger devices at startup, relying on basic (customizable) JSON configuration and systemd for starting up OSDs. + +This process involves two steps: + +#. :ref:`Scan ` the running OSD or the data device +#. :ref:`Activate ` the scanned OSD + +The scanning will infer everything that ``ceph-volume`` needs to start the OSD, +so that when activation is needed, the OSD can start normally without getting +interference from ``ceph-disk``. + +As part of the activation process the systemd units for ``ceph-disk`` in charge +of reacting to ``udev`` events, are linked to ``/dev/null`` so that they are +fully inactive. diff --git a/ceph/doc/cephfs/administration.rst b/ceph/doc/cephfs/administration.rst index e9d919555..71a0e4075 100644 --- a/ceph/doc/cephfs/administration.rst +++ b/ceph/doc/cephfs/administration.rst @@ -74,6 +74,22 @@ appear to be eg. exabytes in size, causing load on the MDS as it tries to enumerate the objects during operations like stats or deletes. +Taking the cluster down +----------------------- + +Taking a CephFS cluster down is done by reducing the number of ranks to 1, +setting the cluster_down flag, and then failing the last rank. For example: + +:: + ceph fs set max_mds 1 + ceph mds deactivate :1 # rank 2 of 2 + ceph status # wait for rank 1 to finish stopping + ceph fs set cluster_down true + ceph mds fail :0 + +Setting the ``cluster_down`` flag prevents standbys from taking over the failed +rank. + Daemons ------- diff --git a/ceph/doc/dev/rbd-export.rst b/ceph/doc/dev/rbd-export.rst index 0d34d611b..2edb637f6 100644 --- a/ceph/doc/dev/rbd-export.rst +++ b/ceph/doc/dev/rbd-export.rst @@ -65,6 +65,14 @@ Image Stripe count - le64: length of appending data (8) - le64: image striping count +ImageMeta Key and Value +----------------------- + +- u8: 'M' +- le64: length of appending data (length of key + length of value + 4 * 2) +- string: image-meta key +- string: image-meta value + Final Record ~~~~~~~~~~~~ @@ -75,9 +83,21 @@ End Diffs records -~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~ + Record the all snapshots and the HEAD in this section. +Snap Protection status +---------------------- + +Record the snapshot's protection status if `--export-format=2`. +- u8: 'p' +- le64: length of appending data (8) +- u8: snap protection status (0 for false, 1 for true) + +Others +------ + - le64: number of diffs - Diffs ... diff --git a/ceph/doc/dev/testing.rst b/ceph/doc/dev/testing.rst new file mode 100644 index 000000000..1d99848a7 --- /dev/null +++ b/ceph/doc/dev/testing.rst @@ -0,0 +1,40 @@ +Testing notes +============= + + +build-integration-branch +------------------------ + +Setup +^^^^^ + +#. Create a github token at ``_ + and put it in ``~/.github_token``. Note that only the + ``public_repo`` under the ``repo`` section needs to be checked. + +#. Create a ceph repo label `wip-yourname-testing` if you don't + already have one at ``_. + +#. Create the ``ci`` remote:: + + git remote add ci git@github.com:ceph/ceph-ci + +Using +^^^^^ + +#. Tag some subset of `needs-qa` commits with your label (usually `wip-yourname-testing`). + +#. Create the integration branch:: + + git checkout master + git pull + ../src/script/build-integration-branch wip-yourname-testing + +#. Smoke test:: + + make && ctest -j12 + +#. Push to ceph-ci:: + + git push ci $(git rev-parse --abbrev-ref HEAD) + diff --git a/ceph/doc/man/8/ceph-fuse.rst b/ceph/doc/man/8/ceph-fuse.rst index cede60e5f..6c88a5a8f 100644 --- a/ceph/doc/man/8/ceph-fuse.rst +++ b/ceph/doc/man/8/ceph-fuse.rst @@ -9,7 +9,7 @@ Synopsis ======== -| **ceph-fuse** [ -m *monaddr*:*port* ] *mountpoint* [ *fuse options* ] +| **ceph-fuse** [-n *client.username*] [ -m *monaddr*:*port* ] *mountpoint* [ *fuse options* ] Description @@ -32,9 +32,13 @@ Options Any options not recognized by ceph-fuse will be passed on to libfuse. +.. option:: -o opt,[opt...] + + mount options. + .. option:: -d - Detach from console and daemonize after startup. + Run in foreground, send all log output to stderr and enable FUSE debugging (-o debug). .. option:: -c ceph.conf, --conf=ceph.conf @@ -45,10 +49,17 @@ Any options not recognized by ceph-fuse will be passed on to libfuse. Connect to specified monitor (instead of looking through ceph.conf). -.. option:: -r root_directory +.. option:: --client_mountpoint/-r root_directory Use root_directory as the mounted root, rather than the full Ceph tree. +.. option:: -f + + Foreground: do not daemonize after startup (run in foreground). Do not generate a pid file. + +.. option:: -s + + Disable multi-threaded operation. Availability ============ diff --git a/ceph/doc/man/8/ceph-volume.rst b/ceph/doc/man/8/ceph-volume.rst index d38cb624e..50a2e5280 100644 --- a/ceph/doc/man/8/ceph-volume.rst +++ b/ceph/doc/man/8/ceph-volume.rst @@ -63,6 +63,10 @@ Optional Arguments: * [--no-systemd] Skip creating and enabling systemd units and starting of OSD services +Multiple OSDs can be activated at once by using the (idempotent) ``--all`` flag:: + + ceph-volume lvm activate --all + **prepare** Prepares a logical volume to be used as an OSD and journal using a ``filestore`` @@ -90,6 +94,10 @@ Required arguments: * --data A logical group name or a path to a logical volume +For encrypting an OSD, the ``--dmcrypt`` flag must be added when preparing +(also supported in the ``create`` sub-command). + + **create** Wraps the two-step process to provision a new osd (calling ``prepare`` first and then ``activate``) into a single one. The reason to prefer ``prepare`` and diff --git a/ceph/doc/man/8/radosgw-admin.rst b/ceph/doc/man/8/radosgw-admin.rst index 4aaeb3922..e4ddb152e 100644 --- a/ceph/doc/man/8/radosgw-admin.rst +++ b/ceph/doc/man/8/radosgw-admin.rst @@ -306,9 +306,13 @@ Options .. option:: --shard-id= - Optional for mdlog list. Required for ``mdlog trim``, + Optional for mdlog list, data sync status. Required for ``mdlog trim``, ``replica mdlog get/delete``, ``replica datalog get/delete``. +.. option:: --max-entries= + + Optional for listing operations to specify the max entires + .. option:: --auth-uid=auid The librados auid. diff --git a/ceph/doc/mgr/balancer.rst b/ceph/doc/mgr/balancer.rst new file mode 100644 index 000000000..191c45593 --- /dev/null +++ b/ceph/doc/mgr/balancer.rst @@ -0,0 +1,146 @@ +Balancer plugin +=============== + +The *balancer* plugin can optimize the placement of PGs across OSDs in +order to achieve a balanced distribution, either automatically or in a +supervised fashion. + +Enabling +-------- + +The *balancer* module is enabled with:: + + ceph mgr module enable balancer + +(It is enabled by default.) + +Status +------ + +The current status of the balancer can be check at any time with:: + + ceph balancer status + + +Automatic balancing +------------------- + +The automatic balancing can be enabled, using the default settings, with:: + + ceph balancer on + +The balancer can be turned back off again with:: + + ceph balancer off + +This will use the ``crush-compat`` mode, which is backward compatible +with older clients, and will make small changes to the data +distribution over time to ensure that OSDs are equally utilized. + + +Throttling +---------- + +No adjustments will be made to the PG distribution if the cluster is +degraded (e.g., because an OSD has failed and the system has not yet +healed itself). + +When the cluster is healthy, the balancer will throttle its changes +such that the percentage of PGs that are misplaced (i.e., that need to +be moved) is below a threshold of (by default) 5%. The +``max_misplaced`` threshold can be adjusted with:: + + ceph config-key set mgr/balancer/max_misplaced .07 # 7% + + +Modes +----- + +There are currently two supported balancer modes: + +#. **crush-compat**. The CRUSH compat mode uses the compat weight-set + feature (introduced in Luminous) to manage an alternative set of + weights for devices in the CRUSH hierarchy. The normal weights + should remain set to the size of the device to reflect the target + amount of data that we want to store on the device. The balancer + then optimizes the weight-set values, adjusting them up or down in + small increments, in order to achieve a distribution that matches + the target distribution as closely as possible. (Because PG + placement is a pseudorandom process, there is a natural amount of + variation in the placement; by optimizing the weights we + counter-act that natural variation.) + + Notably, this mode is *fully backwards compatible* with older + clients: when an OSDMap and CRUSH map is shared with older clients, + we present the optimized weights as the "real" weights. + + The primary restriction of this mode is that the balancer cannot + handle multiple CRUSH hierarchies with different placement rules if + the subtrees of the hierarchy share any OSDs. (This is normally + not the case, and is generally not a recommended configuration + because it is hard to manage the space utilization on the shared + OSDs.) + +#. **upmap**. Starting with Luminous, the OSDMap can store explicit + mappings for individual OSDs as exceptions to the normal CRUSH + placement calculation. These `upmap` entries provide fine-grained + control over the PG mapping. This CRUSH mode will optimize the + placement of individual PGs in order to achieve a balanced + distribution. In most cases, this distribution is "perfect," which + an equal number of PGs on each OSD (+/-1 PG, since they might not + divide evenly). + + Note that using upmap requires that all clients be Luminous or newer. + +The default mode is ``crush-compat``. The mode can be adjusted with:: + + ceph balancer mode upmap + +or:: + + ceph balancer mode crush-compat + +Supervised optimization +----------------------- + +The balancer operation is broken into a few distinct phases: + +#. building a *plan* +#. evaluating the quality of the data distribution, either for the current PG distribution, or the PG distribution that would result after executing a *plan* +#. executing the *plan* + +To evautate and score the current distribution,:: + + ceph balancer eval + +You can also evaluate the distribution for a single pool with:: + + ceph balancer eval + +Greater detail for the evaluation can be seen with:: + + ceph balancer eval-verbose ... + +The balancer can generate a plan, using the currently configured mode, with:: + + ceph balancer optimize + +The name is provided by the user and can be any useful identifying string. The contents of a plan can be seen with:: + + ceph balancer show + +Old plans can be discarded with:: + + ceph balancer rm + +Currently recorded plans are shown as part of the status command:: + + ceph balancer status + +The quality of the distribution that would result after executing a plan can be calculated with:: + + ceph balancer eval + +Assuming the plan is expected to improve the distribution (i.e., it has a lower score than the current cluster state), the user can execute that plan with:: + + ceph balancer execute diff --git a/ceph/doc/mgr/index.rst b/ceph/doc/mgr/index.rst index 53844ba24..30f951606 100644 --- a/ceph/doc/mgr/index.rst +++ b/ceph/doc/mgr/index.rst @@ -27,6 +27,7 @@ sensible. Installation and Configuration Writing plugins + Balancer plugin Dashboard plugin Local pool plugin RESTful plugin diff --git a/ceph/doc/mgr/influx.rst b/ceph/doc/mgr/influx.rst index 37aa5cd63..066c958a0 100644 --- a/ceph/doc/mgr/influx.rst +++ b/ceph/doc/mgr/influx.rst @@ -53,7 +53,8 @@ Additional optional configuration settings are: :interval: Time between reports to InfluxDB. Default 5 seconds. :database: InfluxDB database name. Default "ceph". You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it. :port: InfluxDB server port. Default 8086 - +:ssl: Use https connection for InfluxDB server. Use "true" or "false". Default false +:verify_ssl: Verify https cert for InfluxDB server. Use "true" or "false". Default true --------- Debugging diff --git a/ceph/doc/mgr/prometheus.rst b/ceph/doc/mgr/prometheus.rst index 5bae6a984..eb869d118 100644 --- a/ceph/doc/mgr/prometheus.rst +++ b/ceph/doc/mgr/prometheus.rst @@ -59,13 +59,13 @@ Pools have a ``ceph_pool_metadata`` field like this: :: - ceph_pool_metadata{pool_id="2",name="cephfs_metadata_a"} 0.0 + ceph_pool_metadata{pool_id="2",name="cephfs_metadata_a"} 1.0 OSDs have a ``ceph_osd_metadata`` field like this: :: - ceph_osd_metadata{cluster_addr="172.21.9.34:6802/19096",device_class="ssd",id="0",public_addr="172.21.9.34:6801/19096",weight="1.0"} 0.0 + ceph_osd_metadata{cluster_addr="172.21.9.34:6802/19096",device_class="ssd",ceph_daemon="osd.0",public_addr="172.21.9.34:6801/19096",weight="1.0"} 1.0 Correlating drive statistics with node_exporter @@ -79,34 +79,46 @@ drive statistics, special series are output like this: :: - ceph_disk_occupation{ceph_daemon="osd.0",device="sdd",instance="myhost",job="ceph"} + ceph_disk_occupation{ceph_daemon="osd.0",device="sdd", exported_instance="myhost"} -To use this to get disk statistics by OSD ID, use the ``and on`` syntax -in your prometheus query like this: +To use this to get disk statistics by OSD ID, use either the ``and`` operator or +the ``*`` operator in your prometheus query. All metadata metrics (like `` +ceph_disk_occupation`` have the value 1 so they act neutral with ``*``. Using ``*`` +allows to use ``group_left`` and ``group_right`` grouping modifiers, so that +the resulting metric has additional labels from one side of the query. + +See the +`prometheus documentation`__ for more information about constructing queries. + +__ https://prometheus.io/docs/prometheus/latest/querying/basics + +The goal is to run a query like :: rate(node_disk_bytes_written[30s]) and on (device,instance) ceph_disk_occupation{ceph_daemon="osd.0"} -See the prometheus documentation for more information about constructing -queries. +Out of the box the above query will not return any metrics since the ``instance`` labels of +both metrics don't match. The ``instance`` label of ``ceph_disk_occupation`` +will be the currently active MGR node. -Note that for this mechanism to work, Ceph and node_exporter must agree -about the values of the ``instance`` label. See the following section -for guidance about to to set up Prometheus in a way that sets -``instance`` properly. + The following two section outline two approaches to remedy this. -Configuring Prometheus server -============================= +Use label_replace +================= + +The ``label_replace`` function (cp. +`label_replace documentation `_) +can add a label to, or alter a label of, a metric within a query. -See the prometheus documentation for full details of how to add -scrape endpoints: the notes -in this section are tips on how to configure Prometheus to capture -the Ceph statistics in the most usefully-labelled form. +To correlate an OSD and its disks write rate, the following query can be used: + +:: -This configuration is necessary because Ceph is reporting metrics -from many hosts and services via a single endpoint, and some -metrics that relate to no physical host (such as pool statistics). + label_replace(rate(node_disk_bytes_written[30s]), "exported_instance", "$1", "instance", "(.*):.*") and on (device,exported_instance) ceph_disk_occupation{ceph_daemon="osd.0"} + +Configuring Prometheus server +============================= honor_labels ------------ @@ -115,27 +127,18 @@ To enable Ceph to output properly-labelled data relating to any host, use the ``honor_labels`` setting when adding the ceph-mgr endpoints to your prometheus configuration. -Without this setting, any ``instance`` labels that Ceph outputs, such -as those in ``ceph_disk_occupation`` series, will be overridden -by Prometheus. - -Ceph instance label -------------------- +This allows Ceph to export the proper ``instance`` label without prometheus +overwriting it. Without this setting, Prometheus applies an ``instance`` label +that includes the hostname and port of the endpoint that the series game from. +Because Ceph clusters have multiple manager daemons, this results in an +``instance`` label that changes spuriously when the active manager daemon +changes. -By default, Prometheus applies an ``instance`` label that includes -the hostname and port of the endpoint that the series game from. Because -Ceph clusters have multiple manager daemons, this results in an ``instance`` -label that changes spuriously when the active manager daemon changes. - -Set a custom ``instance`` label in your Prometheus target configuration: -you might wish to set it to the hostname of your first monitor, or something -completely arbitrary like "ceph_cluster". - -node_exporter instance labels +node_exporter hostname labels ----------------------------- Set your ``instance`` labels to match what appears in Ceph's OSD metadata -in the ``hostname`` field. This is generally the short hostname of the node. +in the ``instance`` field. This is generally the short hostname of the node. This is only necessary if you want to correlate Ceph stats with host stats, but you may find it useful to do it in all cases in case you want to do @@ -145,7 +148,8 @@ Example configuration --------------------- This example shows a single node configuration running ceph-mgr and -node_exporter on a server called ``senta04``. +node_exporter on a server called ``senta04``. Note that this requires to add the +appropriate instance label to every ``node_exporter`` target individually. This is just an example: there are other ways to configure prometheus scrape targets and label rewrite rules. @@ -180,9 +184,7 @@ ceph_targets.yml [ { "targets": [ "senta04.mydomain.com:9283" ], - "labels": { - "instance": "ceph_cluster" - } + "labels": {} } ] diff --git a/ceph/doc/rados/configuration/auth-config-ref.rst b/ceph/doc/rados/configuration/auth-config-ref.rst index eb14fa43d..1d72a395d 100644 --- a/ceph/doc/rados/configuration/auth-config-ref.rst +++ b/ceph/doc/rados/configuration/auth-config-ref.rst @@ -291,18 +291,16 @@ You can override these locations, but it is not recommended. Signatures ---------- -In Ceph Bobtail and subsequent versions, we prefer that Ceph authenticate all -ongoing messages between the entities using the session key set up for that -initial authentication. However, Argonaut and earlier Ceph daemons do not know -how to perform ongoing message authentication. To maintain backward -compatibility (e.g., running both Botbail and Argonaut daemons in the same -cluster), message signing is **off** by default. If you are running Bobtail or -later daemons exclusively, configure Ceph to require signatures. +Ceph performs a signature check that provides some limited protection +against messages being tampered with in flight (e.g., by a "man in the +middle" attack). Like other parts of Ceph authentication, Ceph provides fine-grained control so you can enable/disable signatures for service messages between the client and Ceph, and you can enable/disable signatures for messages between Ceph daemons. +Note that even with signatures enabled data is not encrypted in +flight. ``cephx require signatures`` @@ -310,6 +308,10 @@ Ceph, and you can enable/disable signatures for messages between Ceph daemons. traffic between the Ceph Client and the Ceph Storage Cluster, and between daemons comprising the Ceph Storage Cluster. + Ceph Argonaut and Linux kernel versions prior to 3.19 do + not support signatures; if such clients are in use this + option can be turned off to allow them to connect. + :Type: Boolean :Required: No :Default: ``false`` @@ -338,7 +340,7 @@ Ceph, and you can enable/disable signatures for messages between Ceph daemons. ``cephx sign messages`` :Description: If the Ceph version supports message signing, Ceph will sign - all messages so they cannot be spoofed. + all messages so they are more difficult to spoof. :Type: Boolean :Default: ``true`` diff --git a/ceph/doc/rados/deployment/ceph-deploy-mds.rst b/ceph/doc/rados/deployment/ceph-deploy-mds.rst index d2afaec3d..aee5242af 100644 --- a/ceph/doc/rados/deployment/ceph-deploy-mds.rst +++ b/ceph/doc/rados/deployment/ceph-deploy-mds.rst @@ -6,10 +6,6 @@ With ``ceph-deploy``, adding and removing metadata servers is a simple task. You just add or remove one or more metadata servers on the command line with one command. -.. important:: You must deploy at least one metadata server to use CephFS. - There is experimental support for running multiple metadata servers. - Do not run multiple active metadata servers in production. - See `MDS Config Reference`_ for details on configuring metadata servers. diff --git a/ceph/doc/rados/operations/add-or-rm-osds.rst b/ceph/doc/rados/operations/add-or-rm-osds.rst index 59ce4c713..8ad28315a 100644 --- a/ceph/doc/rados/operations/add-or-rm-osds.rst +++ b/ceph/doc/rados/operations/add-or-rm-osds.rst @@ -164,6 +164,7 @@ weight). Note that this practice will no longer be necessary in Bobtail and subsequent releases. +.. _rados-replacing-an-osd: Replacing an OSD ---------------- diff --git a/ceph/doc/rados/operations/pools.rst b/ceph/doc/rados/operations/pools.rst index 8dcccd63d..ae569f92a 100644 --- a/ceph/doc/rados/operations/pools.rst +++ b/ceph/doc/rados/operations/pools.rst @@ -750,6 +750,13 @@ You may get values for the following keys: :Type: Double +``allow_ec_overwrites`` + +:Description: see allow_ec_overwrites_ + +:Type: Boolean + + Set the Number of Object Replicas ================================= diff --git a/ceph/doc/radosgw/frontends.rst b/ceph/doc/radosgw/frontends.rst index cdc5c277d..ff6323ee4 100644 --- a/ceph/doc/radosgw/frontends.rst +++ b/ceph/doc/radosgw/frontends.rst @@ -20,12 +20,25 @@ Options ``port`` -:Description: Sets the listening port number. +:Description: Sets the listening port number. Can be specified multiple + times as in ``port=80 port=8000``. :Type: Integer :Default: ``80`` +``endpoint`` + +:Description: Sets the listening address in the form ``address[:port]``, + where the address is an IPv4 address string in dotted decimal + form, or an IPv6 address in hexadecimal notation. The + optional port defaults to 80. Can be specified multiple times + as in ``endpoint=::1 endpoint=192.168.0.100:8000``. + +:Type: Integer +:Default: None + + Civetweb ======== @@ -43,7 +56,8 @@ Options :Description: Sets the listening port number. For SSL-enabled ports, add an ``s`` suffix like ``443s``. To bind a specific IPv4 or IPv6 address, use the form ``address:port``. Multiple endpoints - can be separated by ``+`` as in ``127.0.0.1:8000+443s``. + can either be separated by ``+`` as in ``127.0.0.1:8000+443s``, + or by providing multiple options as in ``port=8000 port=443s``. :Type: String :Default: ``7480`` diff --git a/ceph/doc/radosgw/keystone.rst b/ceph/doc/radosgw/keystone.rst index 398276c74..4220b75c5 100644 --- a/ceph/doc/radosgw/keystone.rst +++ b/ceph/doc/radosgw/keystone.rst @@ -47,6 +47,13 @@ For a v3 version of the OpenStack Identity API you should replace rgw keystone admin domain = {keystone admin domain name} rgw keystone admin project = {keystone admin project name} +For compatibility with previous versions of ceph, it is also +possible to set ``rgw keystone implicit tenants`` to either +``s3`` or ``swift``. This has the effect of splitting +the identity space such that the indicated protocol will +only use implicit tenants, and the other protocol will +never use implicit tenants. Some older versions of ceph +only supported implicit tenants with swift. Prior to Kilo ------------- diff --git a/ceph/doc/radosgw/multitenancy.rst b/ceph/doc/radosgw/multitenancy.rst index 95f22d7a1..b16f704b3 100644 --- a/ceph/doc/radosgw/multitenancy.rst +++ b/ceph/doc/radosgw/multitenancy.rst @@ -95,6 +95,7 @@ Swift with Keystone TBD -- don't forget to explain the function of rgw keystone implicit tenants = true in commit e9259486decab52a362443d3fd3dec33b0ec654f + [ There is a description of this in keystone.rst ] Notes and known issues ---------------------- diff --git a/ceph/qa/cephfs/clusters/3-mds.yaml b/ceph/qa/cephfs/clusters/3-mds.yaml index 71f39af08..c0d463a90 100644 --- a/ceph/qa/cephfs/clusters/3-mds.yaml +++ b/ceph/qa/cephfs/clusters/3-mds.yaml @@ -1,7 +1,7 @@ roles: - [mon.a, mon.c, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3] - [mon.b, mgr.x, mds.b, mds.c, osd.4, osd.5, osd.6, osd.7] -- [client.0] +- [client.0, client.1] openstack: - volumes: # attached to each instance count: 4 diff --git a/ceph/qa/cephfs/clusters/9-mds.yaml b/ceph/qa/cephfs/clusters/9-mds.yaml index 86be381ee..0bf240272 100644 --- a/ceph/qa/cephfs/clusters/9-mds.yaml +++ b/ceph/qa/cephfs/clusters/9-mds.yaml @@ -1,7 +1,7 @@ roles: - [mon.a, mon.c, mgr.y, mds.a, mds.b, mds.c, mds.d, osd.0, osd.1, osd.2, osd.3] - [mon.b, mgr.x, mds.e, mds.f, mds.g, mds.h, mds.i, osd.4, osd.5, osd.6, osd.7] -- [client.0] +- [client.0, client.1] openstack: - volumes: # attached to each instance count: 4 diff --git a/ceph/qa/distros/all/rhel_7.5.yaml b/ceph/qa/distros/all/rhel_7.5.yaml new file mode 100644 index 000000000..e5aaf3d30 --- /dev/null +++ b/ceph/qa/distros/all/rhel_7.5.yaml @@ -0,0 +1,2 @@ +os_type: rhel +os_version: "7.5" diff --git a/ceph/qa/distros/all/ubuntu_18.04.yaml b/ceph/qa/distros/all/ubuntu_18.04.yaml new file mode 100644 index 000000000..4d4464884 --- /dev/null +++ b/ceph/qa/distros/all/ubuntu_18.04.yaml @@ -0,0 +1,2 @@ +os_type: ubuntu +os_version: "18.04" diff --git a/ceph/qa/overrides/more-active-recovery.yaml b/ceph/qa/overrides/more-active-recovery.yaml new file mode 100644 index 000000000..bfe86e4de --- /dev/null +++ b/ceph/qa/overrides/more-active-recovery.yaml @@ -0,0 +1,6 @@ +overrides: + ceph: + conf: + global: + osd_recovery_max_active: 10 + osd_recovery_max_single_start: 10 diff --git a/ceph/qa/standalone/ceph-helpers.sh b/ceph/qa/standalone/ceph-helpers.sh index 6a8eebff4..b9dd86bf2 100755 --- a/ceph/qa/standalone/ceph-helpers.sh +++ b/ceph/qa/standalone/ceph-helpers.sh @@ -514,6 +514,11 @@ function create_pool() { sleep 1 } +function delete_pool() { + local poolname=$1 + ceph osd pool delete $poolname $poolname --yes-i-really-really-mean-it +} + ####################################################################### function run_mgr() { diff --git a/ceph/qa/standalone/erasure-code/test-erasure-code.sh b/ceph/qa/standalone/erasure-code/test-erasure-code.sh index 6dd5833ad..452161dbd 100755 --- a/ceph/qa/standalone/erasure-code/test-erasure-code.sh +++ b/ceph/qa/standalone/erasure-code/test-erasure-code.sh @@ -62,12 +62,6 @@ function create_erasure_coded_pool() { wait_for_clean || return 1 } -function delete_pool() { - local poolname=$1 - - ceph osd pool delete $poolname $poolname --yes-i-really-really-mean-it -} - function rados_put_get() { local dir=$1 local poolname=$2 diff --git a/ceph/qa/standalone/erasure-code/test-erasure-eio.sh b/ceph/qa/standalone/erasure-code/test-erasure-eio.sh index 5dccebb3b..32bef54ef 100755 --- a/ceph/qa/standalone/erasure-code/test-erasure-eio.sh +++ b/ceph/qa/standalone/erasure-code/test-erasure-eio.sh @@ -80,9 +80,8 @@ function create_erasure_coded_pool() { wait_for_clean || return 1 } -function delete_pool() { +function delete_erasure_coded_pool() { local poolname=$1 - ceph osd pool delete $poolname $poolname --yes-i-really-really-mean-it ceph osd erasure-code-profile rm myprofile } @@ -260,7 +259,7 @@ function TEST_rados_get_subread_eio_shard_0() { # inject eio on primary OSD (0) and replica OSD (1) local shard_id=0 rados_put_get_data eio $dir $shard_id || return 1 - delete_pool $poolname + delete_erasure_coded_pool $poolname } function TEST_rados_get_subread_eio_shard_1() { @@ -272,7 +271,7 @@ function TEST_rados_get_subread_eio_shard_1() { # inject eio into replicas OSD (1) and OSD (2) local shard_id=1 rados_put_get_data eio $dir $shard_id || return 1 - delete_pool $poolname + delete_erasure_coded_pool $poolname } # We don't remove the object from the primary because @@ -287,7 +286,7 @@ function TEST_rados_get_subread_missing() { # inject remove into replicas OSD (1) and OSD (2) local shard_id=1 rados_put_get_data remove $dir $shard_id || return 1 - delete_pool $poolname + delete_erasure_coded_pool $poolname } # @@ -309,7 +308,7 @@ function TEST_rados_get_bad_size_shard_0() { rados_get_data_bad_size $dir $shard_id 10 || return 1 rados_get_data_bad_size $dir $shard_id 0 || return 1 rados_get_data_bad_size $dir $shard_id 256 add || return 1 - delete_pool $poolname + delete_erasure_coded_pool $poolname } function TEST_rados_get_bad_size_shard_1() { @@ -323,7 +322,7 @@ function TEST_rados_get_bad_size_shard_1() { rados_get_data_bad_size $dir $shard_id 10 || return 1 rados_get_data_bad_size $dir $shard_id 0 || return 1 rados_get_data_bad_size $dir $shard_id 256 add || return 1 - delete_pool $poolname + delete_erasure_coded_pool $poolname } function TEST_rados_get_with_subreadall_eio_shard_0() { @@ -337,7 +336,7 @@ function TEST_rados_get_with_subreadall_eio_shard_0() { # inject eio on primary OSD (0) rados_put_get_data eio $dir $shard_id recovery || return 1 - delete_pool $poolname + delete_erasure_coded_pool $poolname } function TEST_rados_get_with_subreadall_eio_shard_1() { @@ -351,11 +350,11 @@ function TEST_rados_get_with_subreadall_eio_shard_1() { # inject eio on replica OSD (1) rados_put_get_data eio $dir $shard_id recovery || return 1 - delete_pool $poolname + delete_erasure_coded_pool $poolname } # Test recovery the first k copies aren't all available -function TEST_ec_recovery_errors() { +function TEST_ec_single_recovery_error() { local dir=$1 local objname=myobject @@ -377,7 +376,102 @@ function TEST_ec_recovery_errors() { # Cluster should recover this object wait_for_clean || return 1 - delete_pool $poolname + rados_get $dir $poolname myobject || return 1 + + delete_erasure_coded_pool $poolname +} + +# Test recovery when repeated reads are needed due to EIO +function TEST_ec_recovery_multiple_errors() { + local dir=$1 + local objname=myobject + + setup_osds 9 || return 1 + + local poolname=pool-jerasure + create_erasure_coded_pool $poolname 4 4 || return 1 + + rados_put $dir $poolname $objname || return 1 + inject_eio ec data $poolname $objname $dir 0 || return 1 + # first read will try shards 0,1,2 when 0 gets EIO, shard 3 gets + # tried as well. Make that fail to test multiple-EIO handling. + inject_eio ec data $poolname $objname $dir 3 || return 1 + inject_eio ec data $poolname $objname $dir 4 || return 1 + + local -a initial_osds=($(get_osds $poolname $objname)) + local last_osd=${initial_osds[-1]} + # Kill OSD + kill_daemons $dir TERM osd.${last_osd} >&2 < /dev/null || return 1 + ceph osd down ${last_osd} || return 1 + ceph osd out ${last_osd} || return 1 + + # Cluster should recover this object + wait_for_clean || return 1 + + rados_get $dir $poolname myobject || return 1 + + delete_erasure_coded_pool $poolname +} + +# Test recovery when there's only one shard to recover, but multiple +# objects recovering in one RecoveryOp +function TEST_ec_recovery_multiple_objects() { + local dir=$1 + local objname=myobject + + ORIG_ARGS=$CEPH_ARGS + CEPH_ARGS+=' --osd-recovery-max-single-start 3 --osd-recovery-max-active 3 ' + setup_osds 7 || return 1 + CEPH_ARGS=$ORIG_ARGS + + local poolname=pool-jerasure + create_erasure_coded_pool $poolname 3 2 || return 1 + + rados_put $dir $poolname test1 + rados_put $dir $poolname test2 + rados_put $dir $poolname test3 + + ceph osd out 0 || return 1 + + # Cluster should recover these objects all at once + wait_for_clean || return 1 + + rados_get $dir $poolname test1 + rados_get $dir $poolname test2 + rados_get $dir $poolname test3 + + delete_erasure_coded_pool $poolname +} + +# test multi-object recovery when the one missing shard gets EIO +function TEST_ec_recovery_multiple_objects_eio() { + local dir=$1 + local objname=myobject + + ORIG_ARGS=$CEPH_ARGS + CEPH_ARGS+=' --osd-recovery-max-single-start 3 --osd-recovery-max-active 3 ' + setup_osds 7 || return 1 + CEPH_ARGS=$ORIG_ARGS + + local poolname=pool-jerasure + create_erasure_coded_pool $poolname 3 2 || return 1 + + rados_put $dir $poolname test1 + rados_put $dir $poolname test2 + rados_put $dir $poolname test3 + + # can't read from this shard anymore + inject_eio ec data $poolname $objname $dir 0 || return 1 + ceph osd out 0 || return 1 + + # Cluster should recover these objects all at once + wait_for_clean || return 1 + + rados_get $dir $poolname test1 + rados_get $dir $poolname test2 + rados_get $dir $poolname test3 + + delete_erasure_coded_pool $poolname } # Test backfill with unfound object @@ -388,9 +482,10 @@ function TEST_ec_backfill_unfound() { # Must be between 1 and $lastobj local testobj=obj250 - export CEPH_ARGS + ORIG_ARGS=$CEPH_ARGS CEPH_ARGS+=' --osd_min_pg_log_entries=5 --osd_max_pg_log_entries=10' setup_osds 5 || return 1 + CEPH_ARGS=$ORIG_ARGS local poolname=pool-jerasure create_erasure_coded_pool $poolname 3 2 || return 1 @@ -455,7 +550,7 @@ function TEST_ec_backfill_unfound() { rm -f ${dir}/ORIGINAL ${dir}/CHECK - delete_pool $poolname + delete_erasure_coded_pool $poolname } # Test recovery with unfound object @@ -466,7 +561,11 @@ function TEST_ec_recovery_unfound() { # Must be between 1 and $lastobj local testobj=obj75 + ORIG_ARGS=$CEPH_ARGS + CEPH_ARGS+=' --osd-recovery-max-single-start 3 --osd-recovery-max-active 3 ' + CEPH_ARGS+=' --osd_min_pg_log_entries=5 --osd_max_pg_log_entries=10' setup_osds 5 || return 1 + CEPH_ARGS=$ORIG_ARGS local poolname=pool-jerasure create_erasure_coded_pool $poolname 3 2 || return 1 @@ -531,7 +630,7 @@ function TEST_ec_recovery_unfound() { rm -f ${dir}/ORIGINAL ${dir}/CHECK - delete_pool $poolname + delete_erasure_coded_pool $poolname } main test-erasure-eio "$@" diff --git a/ceph/qa/standalone/osd/ec-error-rollforward.sh b/ceph/qa/standalone/osd/ec-error-rollforward.sh new file mode 100755 index 000000000..e3a6480a5 --- /dev/null +++ b/ceph/qa/standalone/osd/ec-error-rollforward.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + # Fix port???? + export CEPH_MON="127.0.0.1:7132" # git grep '\<7132\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON --osd-objectstore filestore" + export margin=10 + export objects=200 + export poolname=test + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +function TEST_ec_error_rollforward() { + local dir=$1 + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + run_osd $dir 2 || return 1 + run_osd $dir 3 || return 1 + + ceph osd erasure-code-profile set ec-profile m=2 k=2 crush-failure-domain=osd + ceph osd pool create ec 1 1 erasure ec-profile + + rados -p ec put foo /etc/passwd + + kill -STOP `cat $dir/osd.2.pid` + + rados -p ec rm foo & + sleep 1 + rados -p ec rm a & + rados -p ec rm b & + rados -p ec rm c & + sleep 1 + kill -9 `cat $dir/osd.?.pid` + kill %1 %2 %3 %4 + + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + run_osd $dir 2 || return 1 + run_osd $dir 3 || return 1 + + wait_for_clean || return 1 +} + +main ec-error-rollforward "$@" diff --git a/ceph/qa/standalone/osd/osd-rep-recov-eio.sh b/ceph/qa/standalone/osd/osd-rep-recov-eio.sh index d0fe53836..3a4785663 100755 --- a/ceph/qa/standalone/osd/osd-rep-recov-eio.sh +++ b/ceph/qa/standalone/osd/osd-rep-recov-eio.sh @@ -57,12 +57,6 @@ function get_state() { jq -r ".[] | select(.pgid==\"$pgid\") | .$sname" } -function delete_pool() { - local poolname=$1 - - ceph osd pool delete $poolname $poolname --yes-i-really-really-mean-it -} - function rados_put() { local dir=$1 local poolname=$2 diff --git a/ceph/qa/standalone/osd/repro_long_log.sh b/ceph/qa/standalone/osd/repro_long_log.sh index 1f4ba29bd..93201a2a5 100755 --- a/ceph/qa/standalone/osd/repro_long_log.sh +++ b/ceph/qa/standalone/osd/repro_long_log.sh @@ -138,10 +138,11 @@ function TEST_trim_max_entries() rados -p test rm foo test_log_size $PGID 3 rados -p test rm foo - test_log_size $PGID 4 - + test_log_size $PGID 3 rados -p test rm foo - test_log_size $PGID 2 + test_log_size $PGID 3 + rados -p test rm foo + test_log_size $PGID 3 } main repro-long-log "$@" diff --git a/ceph/qa/standalone/scrub/osd-recovery-scrub.sh b/ceph/qa/standalone/scrub/osd-recovery-scrub.sh index 1061dc77e..a4ddd6b34 100755 --- a/ceph/qa/standalone/scrub/osd-recovery-scrub.sh +++ b/ceph/qa/standalone/scrub/osd-recovery-scrub.sh @@ -127,3 +127,4 @@ main osd-recovery-scrub "$@" # Local Variables: # compile-command: "cd build ; make -j4 && \ # ../qa/run-standalone.sh osd-recovery-scrub.sh" +# End: diff --git a/ceph/qa/standalone/scrub/osd-scrub-repair.sh b/ceph/qa/standalone/scrub/osd-scrub-repair.sh index 52f171ff1..a3732ba32 100755 --- a/ceph/qa/standalone/scrub/osd-scrub-repair.sh +++ b/ceph/qa/standalone/scrub/osd-scrub-repair.sh @@ -56,6 +56,7 @@ function run() { export CEPH_ARGS CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " CEPH_ARGS+="--mon-host=$CEPH_MON " + CEPH_ARGS+="--osd-skip-data-digest=false " local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} for func in $funcs ; do @@ -2719,8 +2720,7 @@ function corrupt_scrub_erasure() { "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -2771,8 +2771,7 @@ function corrupt_scrub_erasure() { "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -2846,8 +2845,7 @@ function corrupt_scrub_erasure() { "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -2953,8 +2951,7 @@ function corrupt_scrub_erasure() { "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -3011,8 +3008,7 @@ function corrupt_scrub_erasure() { "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -3063,8 +3059,7 @@ function corrupt_scrub_erasure() { "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -3122,8 +3117,7 @@ function corrupt_scrub_erasure() { "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -3218,8 +3212,7 @@ function corrupt_scrub_erasure() { "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -3375,8 +3368,7 @@ EOF "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -3430,8 +3422,7 @@ EOF "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -3510,8 +3501,7 @@ EOF "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -3623,8 +3613,7 @@ EOF "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -3685,8 +3674,7 @@ EOF "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -3739,8 +3727,7 @@ EOF "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -3803,8 +3790,7 @@ EOF "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -3900,8 +3886,7 @@ EOF "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -4038,8 +4023,7 @@ EOF "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -4093,8 +4077,7 @@ EOF "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -4174,8 +4157,7 @@ EOF "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -4250,8 +4232,7 @@ EOF "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -4363,8 +4344,7 @@ EOF "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -4423,8 +4403,7 @@ EOF "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -4478,8 +4457,7 @@ EOF "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -4543,8 +4521,7 @@ EOF "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -4642,8 +4619,7 @@ EOF "lost": 0, "flags": [ "dirty", - "data_digest", - "omap_digest" + "data_digest" ], "truncate_seq": 0, "truncate_size": 0, @@ -5134,6 +5110,6 @@ EOF main osd-scrub-repair "$@" # Local Variables: -# compile-command: "cd ../.. ; make -j4 && \ -# test/osd/osd-scrub-repair.sh # TEST_corrupt_and_repair_replicated" +# compile-command: "cd build ; make -j4 && \ +# ../qa/run-standalone.sh osd-scrub-repair.sh" # End: diff --git a/ceph/qa/standalone/scrub/osd-scrub-snaps.sh b/ceph/qa/standalone/scrub/osd-scrub-snaps.sh index cf6f67b6d..a83cfe75c 100755 --- a/ceph/qa/standalone/scrub/osd-scrub-snaps.sh +++ b/ceph/qa/standalone/scrub/osd-scrub-snaps.sh @@ -20,6 +20,9 @@ source $CEPH_ROOT/qa/standalone/ceph-helpers.sh # Set to "yes" in order to ignore diff errors and save results to update test getjson="no" +jqfilter='.inconsistents' +sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)' + function run() { local dir=$1 shift @@ -37,33 +40,12 @@ function run() { done } -function TEST_scrub_snaps() { +function create_scenario() { local dir=$1 - local poolname=test - local OBJS=15 - local OSDS=1 - - TESTDATA="testdata.$$" + local poolname=$2 + local TESTDATA=$3 + local osd=$4 - run_mon $dir a --osd_pool_default_size=$OSDS || return 1 - run_mgr $dir x || return 1 - for osd in $(seq 0 $(expr $OSDS - 1)) - do - run_osd $dir $osd || return 1 - done - - # Create a pool with a single pg - create_pool $poolname 1 1 - wait_for_clean || return 1 - poolid=$(ceph osd dump | grep "^pool.*[']test[']" | awk '{ print $2 }') - - dd if=/dev/urandom of=$TESTDATA bs=1032 count=1 - for i in `seq 1 $OBJS` - do - rados -p $poolname put obj${i} $TESTDATA - done - - local primary=$(get_primary $poolname obj1) SNAP=1 rados -p $poolname mksnap snap${SNAP} dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP} @@ -107,60 +89,91 @@ function TEST_scrub_snaps() { # Don't need to use ceph_objectstore_tool() function because osd stopped - JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj1)" - ceph-objectstore-tool --data-path $dir/${primary} "$JSON" --force remove + JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj1)" + ceph-objectstore-tool --data-path $dir/${osd} "$JSON" --force remove - JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj5 | grep \"snapid\":2)" - ceph-objectstore-tool --data-path $dir/${primary} "$JSON" remove + JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --op list obj5 | grep \"snapid\":2)" + ceph-objectstore-tool --data-path $dir/${osd} "$JSON" remove - JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj5 | grep \"snapid\":1)" + JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --op list obj5 | grep \"snapid\":1)" OBJ5SAVE="$JSON" - ceph-objectstore-tool --data-path $dir/${primary} "$JSON" remove + ceph-objectstore-tool --data-path $dir/${osd} "$JSON" remove - JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj5 | grep \"snapid\":4)" + JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --op list obj5 | grep \"snapid\":4)" dd if=/dev/urandom of=$TESTDATA bs=256 count=18 - ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-bytes $TESTDATA + ceph-objectstore-tool --data-path $dir/${osd} "$JSON" set-bytes $TESTDATA - JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj3)" + JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj3)" dd if=/dev/urandom of=$TESTDATA bs=256 count=15 - ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-bytes $TESTDATA + ceph-objectstore-tool --data-path $dir/${osd} "$JSON" set-bytes $TESTDATA - JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj4 | grep \"snapid\":7)" - ceph-objectstore-tool --data-path $dir/${primary} "$JSON" remove + JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --op list obj4 | grep \"snapid\":7)" + ceph-objectstore-tool --data-path $dir/${osd} "$JSON" remove - JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj2)" - ceph-objectstore-tool --data-path $dir/${primary} "$JSON" rm-attr snapset + JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj2)" + ceph-objectstore-tool --data-path $dir/${osd} "$JSON" rm-attr snapset # Create a clone which isn't in snapset and doesn't have object info JSON="$(echo "$OBJ5SAVE" | sed s/snapid\":1/snapid\":7/)" dd if=/dev/urandom of=$TESTDATA bs=256 count=7 - ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-bytes $TESTDATA + ceph-objectstore-tool --data-path $dir/${osd} "$JSON" set-bytes $TESTDATA - rm -f $TESTDATA - - JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj6)" - ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset - JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj7)" - ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset corrupt - JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj8)" - ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset seq - JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj9)" - ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset clone_size - JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj10)" - ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset clone_overlap - JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj11)" - ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset clones - JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj12)" - ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset head - JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj13)" - ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset snaps - JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj14)" - ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset size + JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj6)" + ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset + JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj7)" + ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset corrupt + JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj8)" + ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset seq + JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj9)" + ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset clone_size + JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj10)" + ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset clone_overlap + JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj11)" + ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset clones + JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj12)" + ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset head + JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj13)" + ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset snaps + JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj14)" + ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset size echo "garbage" > $dir/bad - JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj15)" - ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-attr snapset $dir/bad + JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj15)" + ceph-objectstore-tool --data-path $dir/${osd} "$JSON" set-attr snapset $dir/bad rm -f $dir/bad +} + +function TEST_scrub_snaps() { + local dir=$1 + local poolname=test + local OBJS=15 + local OSDS=1 + + TESTDATA="testdata.$$" + + run_mon $dir a --osd_pool_default_size=$OSDS || return 1 + run_mgr $dir x || return 1 + for osd in $(seq 0 $(expr $OSDS - 1)) + do + run_osd $dir $osd || return 1 + done + + # Create a pool with a single pg + create_pool $poolname 1 1 + wait_for_clean || return 1 + poolid=$(ceph osd dump | grep "^pool.*[']test[']" | awk '{ print $2 }') + + dd if=/dev/urandom of=$TESTDATA bs=1032 count=1 + for i in `seq 1 $OBJS` + do + rados -p $poolname put obj${i} $TESTDATA + done + + local primary=$(get_primary $poolname obj1) + + create_scenario $dir $poolname $TESTDATA $primary + + rm -f $TESTDATA for osd in $(seq 0 $(expr $OSDS - 1)) do @@ -172,7 +185,8 @@ function TEST_scrub_snaps() { cat $dir/osd.0.log return 1 fi - grep 'log_channel' $dir/osd.0.log + + test "$(grep "_scan_snaps start" $dir/osd.${primary}.log | wc -l)" = "2" || return 1 rados list-inconsistent-pg $poolname > $dir/json || return 1 # Check pg count @@ -180,10 +194,22 @@ function TEST_scrub_snaps() { # Check pgid test $(jq -r '.[0]' $dir/json) = $pgid || return 1 - rados list-inconsistent-snapset $pgid > $dir/json || return 1 + rados list-inconsistent-obj $pgid > $dir/json || return 1 - local jqfilter='.inconsistents' - local sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)' + # The injected snapshot errors with a single copy pool doesn't + # see object errors because all the issues are detected by + # comparing copies. + jq "$jqfilter" << EOF | python -c "$sortkeys" > $dir/checkcsjson +{ + "epoch": 17, + "inconsistents": [] +} +EOF + + jq "$jqfilter" $dir/json | python -c "$sortkeys" > $dir/csjson + diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 + + rados list-inconsistent-snapset $pgid > $dir/json || return 1 jq "$jqfilter" << EOF | python -c "$sortkeys" > $dir/checkcsjson { @@ -676,6 +702,13 @@ EOF jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-snap.json || return 1 fi + pidfiles=$(find $dir 2>/dev/null | grep 'osd[^/]*\.pid') + pids="" + for pidfile in ${pidfiles} + do + pids+="$(cat $pidfile) " + done + for i in `seq 1 7` do rados -p $poolname rmsnap snap$i @@ -683,14 +716,14 @@ EOF ERRORS=0 - pidfile=$(find $dir 2>/dev/null | grep $name_prefix'[^/]*\.pid') - pid=$(cat $pidfile) - if ! kill -0 $pid - then - echo "OSD crash occurred" - tail -100 $dir/osd.0.log - ERRORS=$(expr $ERRORS + 1) - fi + for pid in $pids + do + if ! kill -0 $pid + then + echo "OSD Crash occurred" + ERRORS=$(expr $ERRORS + 1) + fi + done kill_daemons $dir || return 1 @@ -739,8 +772,505 @@ EOF return 0 } +function _scrub_snaps_multi() { + local dir=$1 + local poolname=test + local OBJS=15 + local OSDS=2 + local which=$2 + + TESTDATA="testdata.$$" + + run_mon $dir a --osd_pool_default_size=$OSDS || return 1 + run_mgr $dir x || return 1 + for osd in $(seq 0 $(expr $OSDS - 1)) + do + run_osd $dir $osd || return 1 + done + + # Create a pool with a single pg + create_pool $poolname 1 1 + wait_for_clean || return 1 + poolid=$(ceph osd dump | grep "^pool.*[']test[']" | awk '{ print $2 }') + + dd if=/dev/urandom of=$TESTDATA bs=1032 count=1 + for i in `seq 1 $OBJS` + do + rados -p $poolname put obj${i} $TESTDATA + done + + local primary=$(get_primary $poolname obj1) + local replica=$(get_not_primary $poolname obj1) + + eval create_scenario $dir $poolname $TESTDATA \$$which + + rm -f $TESTDATA + + for osd in $(seq 0 $(expr $OSDS - 1)) + do + run_osd $dir $osd || return 1 + done + + local pgid="${poolid}.0" + if ! pg_scrub "$pgid" ; then + cat $dir/osd.0.log + return 1 + fi + + test "$(grep "_scan_snaps start" $dir/osd.${primary}.log | wc -l)" -gt "3" || return 1 + test "$(grep "_scan_snaps start" $dir/osd.${replica}.log | wc -l)" -gt "3" || return 1 + + rados list-inconsistent-pg $poolname > $dir/json || return 1 + # Check pg count + test $(jq '. | length' $dir/json) = "1" || return 1 + # Check pgid + test $(jq -r '.[0]' $dir/json) = $pgid || return 1 + + rados list-inconsistent-obj $pgid --format=json-pretty + + rados list-inconsistent-snapset $pgid > $dir/json || return 1 + + # Since all of the snapshots on the primary is consistent there are no errors here + if [ $which = "replica" ]; + then + scruberrors="21" + jq "$jqfilter" << EOF | python -c "$sortkeys" > $dir/checkcsjson +{ + "epoch": 23, + "inconsistents": [] +} +EOF + +else + scruberrors="33" + jq "$jqfilter" << EOF | python -c "$sortkeys" > $dir/checkcsjson +{ + "epoch": 23, + "inconsistents": [ + { + "name": "obj10", + "nspace": "", + "locator": "", + "snap": 1, + "errors": [ + "size_mismatch" + ] + }, + { + "name": "obj11", + "nspace": "", + "locator": "", + "snap": 1, + "errors": [ + "headless" + ] + }, + { + "name": "obj14", + "nspace": "", + "locator": "", + "snap": 1, + "errors": [ + "size_mismatch" + ] + }, + { + "name": "obj6", + "nspace": "", + "locator": "", + "snap": 1, + "errors": [ + "headless" + ] + }, + { + "name": "obj7", + "nspace": "", + "locator": "", + "snap": 1, + "errors": [ + "headless" + ] + }, + { + "name": "obj9", + "nspace": "", + "locator": "", + "snap": 1, + "errors": [ + "size_mismatch" + ] + }, + { + "name": "obj5", + "nspace": "", + "locator": "", + "snap": 7, + "errors": [ + "info_missing", + "headless" + ] + }, + { + "name": "obj10", + "nspace": "", + "locator": "", + "snap": "head", + "snapset": { + "head_exists": 1, + "snap_context": { + "seq": 1, + "snaps": [ + 1 + ] + }, + "clones": [ + { + "snap": 1, + "size": 1032, + "overlap": "????", + "snaps": [ + 1 + ] + } + ] + }, + "errors": [] + }, + { + "name": "obj11", + "nspace": "", + "locator": "", + "snap": "head", + "snapset": { + "head_exists": 1, + "snap_context": { + "seq": 1, + "snaps": [ + 1 + ] + }, + "clones": [] + }, + "errors": [ + "extra_clones" + ], + "extra clones": [ + 1 + ] + }, + { + "errors": [ + "head_mismatch" + ], + "locator": "", + "name": "obj12", + "nspace": "", + "snap": "head", + "snapset": { + "clones": [ + { + "overlap": "[]", + "size": 1032, + "snap": 1, + "snaps": [ + 1 + ] + } + ], + "head_exists": 0, + "snap_context": { + "seq": 1, + "snaps": [ + 1 + ] + } + } + }, + { + "name": "obj14", + "nspace": "", + "locator": "", + "snap": "head", + "snapset": { + "head_exists": 1, + "snap_context": { + "seq": 1, + "snaps": [ + 1 + ] + }, + "clones": [ + { + "snap": 1, + "size": 1033, + "overlap": "[]", + "snaps": [ + 1 + ] + } + ] + }, + "errors": [] + }, + { + "name": "obj5", + "nspace": "", + "locator": "", + "snap": "head", + "snapset": { + "head_exists": 1, + "snap_context": { + "seq": 6, + "snaps": [ + 6, + 5, + 4, + 3, + 2, + 1 + ] + }, + "clones": [ + { + "snap": 1, + "size": 1032, + "overlap": "[]", + "snaps": [ + 1 + ] + }, + { + "snap": 2, + "size": 256, + "overlap": "[]", + "snaps": [ + 2 + ] + }, + { + "snap": 4, + "size": 512, + "overlap": "[]", + "snaps": [ + 4, + 3 + ] + }, + { + "snap": 6, + "size": 1024, + "overlap": "[]", + "snaps": [ + 6, + 5 + ] + } + ] + }, + "errors": [ + "extra_clones" + ], + "extra clones": [ + 7 + ] + }, + { + "name": "obj6", + "nspace": "", + "locator": "", + "snap": "head", + "snapset": { + "head_exists": 1, + "snap_context": { + "seq": 1, + "snaps": [ + 1 + ] + }, + "clones": [] + }, + "errors": [ + "extra_clones" + ], + "extra clones": [ + 1 + ] + }, + { + "name": "obj7", + "nspace": "", + "locator": "", + "snap": "head", + "snapset": { + "head_exists": 0, + "snap_context": { + "seq": 0, + "snaps": [] + }, + "clones": [] + }, + "errors": [ + "head_mismatch", + "extra_clones" + ], + "extra clones": [ + 1 + ] + }, + { + "name": "obj8", + "nspace": "", + "locator": "", + "snap": "head", + "snapset": { + "head_exists": 1, + "snap_context": { + "seq": 0, + "snaps": [ + 1 + ] + }, + "clones": [ + { + "snap": 1, + "size": 1032, + "overlap": "[]", + "snaps": [ + 1 + ] + } + ] + }, + "errors": [ + "snapset_error" + ] + }, + { + "name": "obj9", + "nspace": "", + "locator": "", + "snap": "head", + "snapset": { + "head_exists": 1, + "snap_context": { + "seq": 1, + "snaps": [ + 1 + ] + }, + "clones": [ + { + "snap": 1, + "size": "????", + "overlap": "[]", + "snaps": [ + 1 + ] + } + ] + }, + "errors": [] + } + ] +} +EOF +fi + + jq "$jqfilter" $dir/json | python -c "$sortkeys" > $dir/csjson + diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 + if test $getjson = "yes" + then + jq '.' $dir/json > save1.json + fi + + if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null; + then + jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-snap.json || return 1 + fi + + pidfiles=$(find $dir 2>/dev/null | grep 'osd[^/]*\.pid') + pids="" + for pidfile in ${pidfiles} + do + pids+="$(cat $pidfile) " + done + + # When removing snapshots with a corrupt replica, it crashes. + # See http://tracker.ceph.com/issues/23875 + if [ $which = "primary" ]; + then + for i in `seq 1 7` + do + rados -p $poolname rmsnap snap$i + done + fi + + ERRORS=0 + + for pid in $pids + do + if ! kill -0 $pid + then + echo "OSD Crash occurred" + ERRORS=$(expr $ERRORS + 1) + fi + done + + kill_daemons $dir || return 1 + + declare -a err_strings + err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1] missing .*:::obj4:7" + err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1]: soid .*:::obj3:head size 3840 != size 768 from auth oi" + err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1] missing .*:::obj5:1" + err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1] missing .*:::obj5:2" + err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1]: soid .*:::obj5:4 size 4608 != size 512 from auth oi" + err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid .*:::obj5:7: failed to pick suitable object info" + err_strings[6]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1] missing .*:::obj1:head" + err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub ${scruberrors} errors" + + for err_string in "${err_strings[@]}" + do + if ! grep "$err_string" $dir/osd.${primary}.log > /dev/null; + then + echo "Missing log message '$err_string'" + ERRORS=$(expr $ERRORS + 1) + fi + done + + if [ $ERRORS != "0" ]; + then + echo "TEST FAILED WITH $ERRORS ERRORS" + return 1 + fi + + echo "TEST PASSED" + return 0 +} + +function TEST_scrub_snaps_replica() { + local dir=$1 + ORIG_ARGS=$CEPH_ARGS + CEPH_ARGS+=" --osd_scrub_chunk_min=3 --osd_scrub_chunk_max=3" + _scrub_snaps_multi $dir replica + err=$? + CEPH_ARGS=$ORIG_ARGS + return $err +} + +function TEST_scrub_snaps_primary() { + local dir=$1 + ORIG_ARGS=$CEPH_ARGS + CEPH_ARGS+=" --osd_scrub_chunk_min=3 --osd_scrub_chunk_max=3" + _scrub_snaps_multi $dir primary + err=$? + CEPH_ARGS=$ORIG_ARGS + return $err +} + main osd-scrub-snaps "$@" # Local Variables: -# compile-command: "cd ../.. ; make -j4 && \ -# test/osd/osd-scrub-snaps.sh" +# compile-command: "cd build ; make -j4 && \ +# ../qa/run-standalone.sh osd-scrub-snaps.sh" +# End: diff --git a/ceph/qa/standalone/scrub/osd-scrub-test.sh b/ceph/qa/standalone/scrub/osd-scrub-test.sh index e6c93684a..b73906af9 100755 --- a/ceph/qa/standalone/scrub/osd-scrub-test.sh +++ b/ceph/qa/standalone/scrub/osd-scrub-test.sh @@ -109,5 +109,6 @@ function TEST_scrub_test() { main osd-scrub-test "$@" # Local Variables: -# compile-command: "cd ../.. ; make -j4 && \ -# qa/standalone/scrub/osd-scrub-test.sh" +# compile-command: "cd build ; make -j4 && \ +# ../qa/run-standalone.sh osd-scrub-test.sh" +# End: diff --git a/ceph/qa/suites/ceph-deploy/basic/tasks/ceph-admin-commands.yaml b/ceph/qa/suites/ceph-deploy/basic/tasks/ceph-admin-commands.yaml index fc4873cee..4e516ab41 100644 --- a/ceph/qa/suites/ceph-deploy/basic/tasks/ceph-admin-commands.yaml +++ b/ceph/qa/suites/ceph-deploy/basic/tasks/ceph-admin-commands.yaml @@ -1,3 +1,9 @@ +openstack: + - machine: + disk: 10 + volumes: + count: 2 + size: 20 roles: - - mon.a - mgr.x @@ -6,14 +12,6 @@ roles: - - osd.1 - mon.b - client.0 -openstack: - - machine: - disk: 10 # GB - ram: 2000 # MB - cpus: 1 - volumes: # attached to each instance - count: 2 - size: 10 # GB tasks: - ssh_keys: - print: "**** done ssh_keys" diff --git a/ceph/qa/suites/ceph-deploy/ceph-volume/cluster/4node.yaml b/ceph/qa/suites/ceph-deploy/ceph-volume/cluster/4node.yaml index 837b1029b..bf4a7f986 100644 --- a/ceph/qa/suites/ceph-deploy/ceph-volume/cluster/4node.yaml +++ b/ceph/qa/suites/ceph-deploy/ceph-volume/cluster/4node.yaml @@ -2,6 +2,12 @@ overrides: ansible.cephlab: vars: quick_lvs_to_create: 4 +openstack: + - machine: + disk: 10 + volumes: + count: 4 + size: 20 roles: - [mon.a, mgr.y, osd.0, osd.1] - [mon.b, osd.2, osd.3] diff --git a/ceph/qa/suites/fs/basic_functional/tasks/cephfs_scrub_tests.yaml b/ceph/qa/suites/fs/basic_functional/tasks/cephfs_scrub_tests.yaml index f79784c0b..7fdee84b1 100644 --- a/ceph/qa/suites/fs/basic_functional/tasks/cephfs_scrub_tests.yaml +++ b/ceph/qa/suites/fs/basic_functional/tasks/cephfs_scrub_tests.yaml @@ -6,7 +6,7 @@ overrides: - Metadata damage detected - bad backtrace on inode - overall HEALTH_ - - (MDS_TRIM) + - \(MDS_TRIM\) conf: mds: mds log max segments: 1 diff --git a/ceph/qa/suites/fs/basic_functional/tasks/client-recovery.yaml b/ceph/qa/suites/fs/basic_functional/tasks/client-recovery.yaml index f5e9a0b5c..a2f56299b 100644 --- a/ceph/qa/suites/fs/basic_functional/tasks/client-recovery.yaml +++ b/ceph/qa/suites/fs/basic_functional/tasks/client-recovery.yaml @@ -10,5 +10,6 @@ overrides: tasks: - cephfs_test_runner: + fail_on_skip: false modules: - tasks.cephfs.test_client_recovery diff --git a/ceph/qa/suites/upgrade/luminous-x/point-to-point-x/% b/ceph/qa/suites/fs/bugs/client_trim_caps/% similarity index 100% rename from ceph/qa/suites/upgrade/luminous-x/point-to-point-x/% rename to ceph/qa/suites/fs/bugs/client_trim_caps/% diff --git a/ceph/qa/suites/fs/bugs/client_trim_caps/begin.yaml b/ceph/qa/suites/fs/bugs/client_trim_caps/begin.yaml new file mode 120000 index 000000000..3279455ce --- /dev/null +++ b/ceph/qa/suites/fs/bugs/client_trim_caps/begin.yaml @@ -0,0 +1 @@ +../../../../cephfs/begin.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/bugs/client_trim_caps/clusters/small-cluster.yaml b/ceph/qa/suites/fs/bugs/client_trim_caps/clusters/small-cluster.yaml new file mode 100644 index 000000000..12047bd7a --- /dev/null +++ b/ceph/qa/suites/fs/bugs/client_trim_caps/clusters/small-cluster.yaml @@ -0,0 +1,9 @@ +roles: +- [mon.a, mgr.x, osd.0, osd.1, osd.2, mds.a, mds.b, client.0] +openstack: +- volumes: # attached to each instance + count: 2 + size: 10 # GB +log-rotate: + ceph-mds: 10G + ceph-osd: 10G diff --git a/ceph/qa/suites/fs/bugs/client_trim_caps/objectstore/bluestore.yaml b/ceph/qa/suites/fs/bugs/client_trim_caps/objectstore/bluestore.yaml new file mode 120000 index 000000000..1728accf9 --- /dev/null +++ b/ceph/qa/suites/fs/bugs/client_trim_caps/objectstore/bluestore.yaml @@ -0,0 +1 @@ +../../../../../cephfs/objectstore-ec/bluestore.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/bugs/client_trim_caps/overrides/+ b/ceph/qa/suites/fs/bugs/client_trim_caps/overrides/+ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/fs/bugs/client_trim_caps/overrides/debug.yaml b/ceph/qa/suites/fs/bugs/client_trim_caps/overrides/debug.yaml new file mode 120000 index 000000000..4fdb9dd12 --- /dev/null +++ b/ceph/qa/suites/fs/bugs/client_trim_caps/overrides/debug.yaml @@ -0,0 +1 @@ +../../../../../cephfs/overrides/debug.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/bugs/client_trim_caps/overrides/frag_enable.yaml b/ceph/qa/suites/fs/bugs/client_trim_caps/overrides/frag_enable.yaml new file mode 120000 index 000000000..9e0f15fc2 --- /dev/null +++ b/ceph/qa/suites/fs/bugs/client_trim_caps/overrides/frag_enable.yaml @@ -0,0 +1 @@ +../../../../../cephfs/overrides/frag_enable.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/bugs/client_trim_caps/overrides/no_client_pidfile.yaml b/ceph/qa/suites/fs/bugs/client_trim_caps/overrides/no_client_pidfile.yaml new file mode 120000 index 000000000..46263867b --- /dev/null +++ b/ceph/qa/suites/fs/bugs/client_trim_caps/overrides/no_client_pidfile.yaml @@ -0,0 +1 @@ +../../../../../overrides/no_client_pidfile.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/bugs/client_trim_caps/overrides/whitelist_health.yaml b/ceph/qa/suites/fs/bugs/client_trim_caps/overrides/whitelist_health.yaml new file mode 120000 index 000000000..42fa3ea7a --- /dev/null +++ b/ceph/qa/suites/fs/bugs/client_trim_caps/overrides/whitelist_health.yaml @@ -0,0 +1 @@ +../../../../../cephfs/overrides/whitelist_health.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/bugs/client_trim_caps/overrides/whitelist_wrongly_marked_down.yaml b/ceph/qa/suites/fs/bugs/client_trim_caps/overrides/whitelist_wrongly_marked_down.yaml new file mode 120000 index 000000000..3728aacfd --- /dev/null +++ b/ceph/qa/suites/fs/bugs/client_trim_caps/overrides/whitelist_wrongly_marked_down.yaml @@ -0,0 +1 @@ +../../../../../cephfs/overrides/whitelist_wrongly_marked_down.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/bugs/client_trim_caps/tasks/trim-i22073.yaml b/ceph/qa/suites/fs/bugs/client_trim_caps/tasks/trim-i22073.yaml new file mode 100644 index 000000000..410606225 --- /dev/null +++ b/ceph/qa/suites/fs/bugs/client_trim_caps/tasks/trim-i22073.yaml @@ -0,0 +1,20 @@ +# Note this test is unlikely to exercise the code as expected in the future: +# "It's too tricky to arrange inodes in session->caps. we don't know if it +# still works in the future." -Zheng + +overrides: + ceph: + log-whitelist: + - MDS cache is too large + - \(MDS_CACHE_OVERSIZED\) +tasks: +- exec: + mon.a: + - "ceph tell mds.* config set mds_max_ratio_caps_per_client 1" + - "ceph tell mds.* config set mds_min_caps_per_client 1" +- background_exec: + mon.a: + - "sleep 30 && ceph tell mds.* config set mds_cache_memory_limit 1" +- exec: + client.0: + - ceph_test_trim_caps diff --git a/ceph/qa/suites/kcephfs/recovery/tasks/client-recovery.yaml b/ceph/qa/suites/kcephfs/recovery/tasks/client-recovery.yaml index 72ce013fa..725a259d2 100644 --- a/ceph/qa/suites/kcephfs/recovery/tasks/client-recovery.yaml +++ b/ceph/qa/suites/kcephfs/recovery/tasks/client-recovery.yaml @@ -10,5 +10,6 @@ overrides: tasks: - cephfs_test_runner: + fail_on_skip: false modules: - tasks.cephfs.test_client_recovery diff --git a/ceph/qa/suites/powercycle/osd/tasks/rados_api_tests.yaml b/ceph/qa/suites/powercycle/osd/tasks/rados_api_tests.yaml index 06f3f5738..52fc991c1 100644 --- a/ceph/qa/suites/powercycle/osd/tasks/rados_api_tests.yaml +++ b/ceph/qa/suites/powercycle/osd/tasks/rados_api_tests.yaml @@ -3,6 +3,7 @@ overrides: log-whitelist: - reached quota - \(POOL_APP_NOT_ENABLED\) + - \(PG_AVAILABILITY\) tasks: - ceph-fuse: - workunit: diff --git a/ceph/qa/suites/rados/basic-luminous/scrub_test.yaml b/ceph/qa/suites/rados/basic-luminous/scrub_test.yaml index d87f5bfdd..39821ad86 100644 --- a/ceph/qa/suites/rados/basic-luminous/scrub_test.yaml +++ b/ceph/qa/suites/rados/basic-luminous/scrub_test.yaml @@ -16,13 +16,14 @@ overrides: - 'deep-scrub 1 missing, 0 inconsistent objects' - 'failed to pick suitable auth object' - overall HEALTH_ - - (OSDMAP_FLAGS) - - (OSD_ - - (PG_ - - (OSD_SCRUB_ERRORS) - - (TOO_FEW_PGS) + - \(OSDMAP_FLAGS\) + - \(OSD_ + - \(PG_ + - \(OSD_SCRUB_ERRORS\) + - \(TOO_FEW_PGS\) conf: osd: osd deep scrub update digest min age: 0 + osd skip data digest: false tasks: - scrub_test: diff --git a/ceph/qa/suites/rados/basic/tasks/rados_api_tests.yaml b/ceph/qa/suites/rados/basic/tasks/rados_api_tests.yaml index 316119c9c..6cebde391 100644 --- a/ceph/qa/suites/rados/basic/tasks/rados_api_tests.yaml +++ b/ceph/qa/suites/rados/basic/tasks/rados_api_tests.yaml @@ -4,11 +4,12 @@ overrides: - reached quota - but it is still running - overall HEALTH_ - - (POOL_FULL) - - (SMALLER_PGP_NUM) - - (CACHE_POOL_NO_HIT_SET) - - (CACHE_POOL_NEAR_FULL) - - (POOL_APP_NOT_ENABLED) + - \(POOL_FULL\) + - \(SMALLER_PGP_NUM\) + - \(CACHE_POOL_NO_HIT_SET\) + - \(CACHE_POOL_NEAR_FULL\) + - \(POOL_APP_NOT_ENABLED\) + - \(PG_AVAILABILITY\) tasks: - workunit: clients: diff --git a/ceph/qa/suites/rados/basic/tasks/repair_test.yaml b/ceph/qa/suites/rados/basic/tasks/repair_test.yaml index 9afbc0417..da765b018 100644 --- a/ceph/qa/suites/rados/basic/tasks/repair_test.yaml +++ b/ceph/qa/suites/rados/basic/tasks/repair_test.yaml @@ -18,9 +18,9 @@ overrides: - attr name mismatch - Regular scrub request, deep-scrub details will be lost - overall HEALTH_ - - (OSDMAP_FLAGS) - - (OSD_ - - (PG_ + - \(OSDMAP_FLAGS\) + - \(OSD_ + - \(PG_ conf: osd: filestore debug inject read err: true diff --git a/ceph/qa/suites/rados/monthrash/ceph.yaml b/ceph/qa/suites/rados/monthrash/ceph.yaml index 9c08e93f3..d7e09a637 100644 --- a/ceph/qa/suites/rados/monthrash/ceph.yaml +++ b/ceph/qa/suites/rados/monthrash/ceph.yaml @@ -6,7 +6,6 @@ overrides: paxos service trim min: 5 # thrashing monitors may make mgr have trouble w/ its keepalive log-whitelist: - - daemon x is unresponsive - overall HEALTH_ - \(MGR_DOWN\) tasks: diff --git a/ceph/qa/suites/rados/multimon/tasks/mon_clock_no_skews.yaml b/ceph/qa/suites/rados/multimon/tasks/mon_clock_no_skews.yaml index ec761e295..a4cea8f3a 100644 --- a/ceph/qa/suites/rados/multimon/tasks/mon_clock_no_skews.yaml +++ b/ceph/qa/suites/rados/multimon/tasks/mon_clock_no_skews.yaml @@ -6,6 +6,6 @@ tasks: - .*clock.*skew.* - clocks not synchronized - overall HEALTH_ - - (MON_CLOCK_SKEW) + - \(MON_CLOCK_SKEW\) - mon_clock_skew_check: expect-skew: false diff --git a/ceph/qa/suites/rados/objectstore/objectstore.yaml b/ceph/qa/suites/rados/objectstore/objectstore.yaml index b5442342b..d68270a8e 100644 --- a/ceph/qa/suites/rados/objectstore/objectstore.yaml +++ b/ceph/qa/suites/rados/objectstore/objectstore.yaml @@ -8,5 +8,5 @@ tasks: - install: - exec: client.0: - - mkdir $TESTDIR/ostest && cd $TESTDIR/ostest && ulimit -c 0 && ulimit -Sn 4096 && ceph_test_objectstore --gtest_filter=-*/3 - - rm -rf $TESTDIR/ostest + - mkdir $TESTDIR/archive/ostest && cd $TESTDIR/archive/ostest && ulimit -Sn 16384 && CEPH_ARGS="--no-log-to-stderr --log-file $TESTDIR/archive/ceph_test_objectstore.log --debug-filestore 20 --debug-bluestore 20" ceph_test_objectstore --gtest_filter=-*/3 --gtest_catch_exceptions=0 + - rm -rf $TESTDIR/archive/ostest diff --git a/ceph/qa/suites/rados/singleton-bluestore/all/cephtool.yaml b/ceph/qa/suites/rados/singleton-bluestore/all/cephtool.yaml index 77eed22d4..b77aa7b7e 100644 --- a/ceph/qa/suites/rados/singleton-bluestore/all/cephtool.yaml +++ b/ceph/qa/suites/rados/singleton-bluestore/all/cephtool.yaml @@ -22,10 +22,20 @@ tasks: - failsafe engaged, dropping updates - failsafe disengaged, no longer dropping updates - overall HEALTH_ - - (OSDMAP_FLAGS) - - (OSD_ - - (PG_ - - (SMALLER_PG_NUM) + - \(OSDMAP_FLAGS\) + - \(OSD_ + - \(PG_ + - \(SMALLER_PG_NUM\) + - \(CACHE_POOL_NO_HIT_SET\) + - erasure code profile property .ruleset-failure-domain. is no longer supported + - \(CACHE_POOL_NEAR_FULL\) + - \(FS_WITH_FAILED_MDS\) + - \(FS_DEGRADED\) + - \(POOL_BACKFILLFULL\) + - \(POOL_FULL\) + - \(SMALLER_PGP_NUM\) + - \(POOL_NEARFULL\) + - \(POOL_APP_NOT_ENABLED\) - workunit: clients: all: diff --git a/ceph/qa/suites/rados/singleton-nomsgr/all/admin_socket_output.yaml b/ceph/qa/suites/rados/singleton-nomsgr/all/admin_socket_output.yaml index bbf330b0b..dfe35bc44 100644 --- a/ceph/qa/suites/rados/singleton-nomsgr/all/admin_socket_output.yaml +++ b/ceph/qa/suites/rados/singleton-nomsgr/all/admin_socket_output.yaml @@ -6,10 +6,10 @@ overrides: - MDS in read-only mode - force file system read-only - overall HEALTH_ - - (OSDMAP_FLAGS) - - (OSD_FULL) - - (MDS_READ_ONLY) - - (POOL_FULL) + - \(OSDMAP_FLAGS\) + - \(OSD_FULL\) + - \(MDS_READ_ONLY\) + - \(POOL_FULL\) tasks: - install: - ceph: diff --git a/ceph/qa/suites/rados/singleton-nomsgr/all/large-omap-object-warnings.yaml b/ceph/qa/suites/rados/singleton-nomsgr/all/large-omap-object-warnings.yaml new file mode 100644 index 000000000..e00a93d0c --- /dev/null +++ b/ceph/qa/suites/rados/singleton-nomsgr/all/large-omap-object-warnings.yaml @@ -0,0 +1,22 @@ +roles: +- [mon.a, mgr.x, osd.0, osd.1, client.0] +overrides: + ceph: + log-whitelist: + - \(OSDMAP_FLAGS\) + - \(OSD_FULL\) + - \(MDS_READ_ONLY\) + - large omap objects + - Large omap object found + - application not enabled + conf: + osd: + osd deep scrub large omap object value sum threshold: 8800000 + osd deep scrub large omap object key threshold: 20000 +tasks: +- install: +- ceph: +- workunit: + clients: + all: + - rados/test_large_omap_detection.py diff --git a/ceph/qa/suites/rados/singleton/all/divergent_priors.yaml b/ceph/qa/suites/rados/singleton/all/divergent_priors.yaml index 00fd13c82..743d73d4a 100644 --- a/ceph/qa/suites/rados/singleton/all/divergent_priors.yaml +++ b/ceph/qa/suites/rados/singleton/all/divergent_priors.yaml @@ -19,9 +19,6 @@ overrides: - \(PG_ - \(OBJECT_ - \(POOL_APP_NOT_ENABLED\) - conf: - osd: - debug osd: 5 tasks: - install: diff --git a/ceph/qa/suites/rados/singleton/all/divergent_priors2.yaml b/ceph/qa/suites/rados/singleton/all/divergent_priors2.yaml index 57237b5a2..2da2c4660 100644 --- a/ceph/qa/suites/rados/singleton/all/divergent_priors2.yaml +++ b/ceph/qa/suites/rados/singleton/all/divergent_priors2.yaml @@ -19,9 +19,6 @@ overrides: - \(PG_ - \(OBJECT_ - \(POOL_APP_NOT_ENABLED\) - conf: - osd: - debug osd: 5 tasks: - install: diff --git a/ceph/qa/suites/rados/singleton/all/osd-recovery-incomplete.yaml b/ceph/qa/suites/rados/singleton/all/osd-recovery-incomplete.yaml index ed5b216b2..a63400be3 100644 --- a/ceph/qa/suites/rados/singleton/all/osd-recovery-incomplete.yaml +++ b/ceph/qa/suites/rados/singleton/all/osd-recovery-incomplete.yaml @@ -21,6 +21,7 @@ tasks: - \(OSD_ - \(PG_ - \(OBJECT_ + - \(REQUEST_SLOW\) conf: osd: osd min pg log entries: 5 diff --git a/ceph/qa/suites/rados/singleton/all/osd-recovery.yaml b/ceph/qa/suites/rados/singleton/all/osd-recovery.yaml index 634e8843e..5479f79b7 100644 --- a/ceph/qa/suites/rados/singleton/all/osd-recovery.yaml +++ b/ceph/qa/suites/rados/singleton/all/osd-recovery.yaml @@ -21,6 +21,7 @@ tasks: - \(PG_ - \(OBJECT_DEGRADED\) - \(SLOW_OPS\) + - \(REQUEST_SLOW\) conf: osd: osd min pg log entries: 5 diff --git a/ceph/qa/suites/rados/singleton/all/pg-removal-interruption.yaml b/ceph/qa/suites/rados/singleton/all/pg-removal-interruption.yaml index 10f18e2ea..3ada5518f 100644 --- a/ceph/qa/suites/rados/singleton/all/pg-removal-interruption.yaml +++ b/ceph/qa/suites/rados/singleton/all/pg-removal-interruption.yaml @@ -16,9 +16,9 @@ tasks: - but it is still running - slow request - overall HEALTH_ - - (OSDMAP_FLAGS) - - (OSD_ - - (PG_ + - \(OSDMAP_FLAGS\) + - \(OSD_ + - \(PG_ - exec: client.0: - sudo ceph osd pool create foo 128 128 diff --git a/ceph/qa/suites/rados/singleton/all/random-eio.yaml b/ceph/qa/suites/rados/singleton/all/random-eio.yaml index 96b2208d1..71e85649e 100644 --- a/ceph/qa/suites/rados/singleton/all/random-eio.yaml +++ b/ceph/qa/suites/rados/singleton/all/random-eio.yaml @@ -19,8 +19,8 @@ tasks: - missing primary copy of - objects unfound and apparently lost - overall HEALTH_ - - (POOL_APP_NOT_ENABLED) - - (PG_DEGRADED) + - \(POOL_APP_NOT_ENABLED\) + - \(PG_DEGRADED\) - full_sequential: - exec: client.0: diff --git a/ceph/qa/suites/rados/singleton/all/thrash-eio.yaml b/ceph/qa/suites/rados/singleton/all/thrash-eio.yaml index cac3cb3b7..e184d911d 100644 --- a/ceph/qa/suites/rados/singleton/all/thrash-eio.yaml +++ b/ceph/qa/suites/rados/singleton/all/thrash-eio.yaml @@ -25,11 +25,11 @@ tasks: - missing primary copy of - objects unfound and apparently lost - overall HEALTH_ - - (OSDMAP_FLAGS) - - (REQUEST_SLOW) - - (PG_ + - \(OSDMAP_FLAGS\) + - \(REQUEST_SLOW\) + - \(PG_ - \(OBJECT_MISPLACED\) - - (OSD_ + - \(OSD_ - thrashosds: op_delay: 30 clean_interval: 120 diff --git a/ceph/qa/suites/rados/singleton/all/thrash_cache_writeback_proxy_none.yaml b/ceph/qa/suites/rados/singleton/all/thrash_cache_writeback_proxy_none.yaml index 82c9b2d0c..c0b270758 100644 --- a/ceph/qa/suites/rados/singleton/all/thrash_cache_writeback_proxy_none.yaml +++ b/ceph/qa/suites/rados/singleton/all/thrash_cache_writeback_proxy_none.yaml @@ -19,7 +19,7 @@ tasks: - but it is still running - slow request - overall HEALTH_ - - (CACHE_POOL_ + - \(CACHE_POOL_ - exec: client.0: - sudo ceph osd pool create base 4 diff --git a/ceph/qa/suites/rados/thrash-erasure-code-big/recovery-overrides b/ceph/qa/suites/rados/thrash-erasure-code-big/recovery-overrides new file mode 120000 index 000000000..1957f2c42 --- /dev/null +++ b/ceph/qa/suites/rados/thrash-erasure-code-big/recovery-overrides @@ -0,0 +1 @@ +../thrash/2-recovery-overrides \ No newline at end of file diff --git a/ceph/qa/suites/rados/thrash-erasure-code-isa/recovery-overrides b/ceph/qa/suites/rados/thrash-erasure-code-isa/recovery-overrides new file mode 120000 index 000000000..1957f2c42 --- /dev/null +++ b/ceph/qa/suites/rados/thrash-erasure-code-isa/recovery-overrides @@ -0,0 +1 @@ +../thrash/2-recovery-overrides \ No newline at end of file diff --git a/ceph/qa/suites/rados/thrash-erasure-code-overwrites/recovery-overrides b/ceph/qa/suites/rados/thrash-erasure-code-overwrites/recovery-overrides new file mode 120000 index 000000000..1957f2c42 --- /dev/null +++ b/ceph/qa/suites/rados/thrash-erasure-code-overwrites/recovery-overrides @@ -0,0 +1 @@ +../thrash/2-recovery-overrides \ No newline at end of file diff --git a/ceph/qa/suites/rados/thrash-erasure-code-shec/recovery-overrides b/ceph/qa/suites/rados/thrash-erasure-code-shec/recovery-overrides new file mode 120000 index 000000000..1957f2c42 --- /dev/null +++ b/ceph/qa/suites/rados/thrash-erasure-code-shec/recovery-overrides @@ -0,0 +1 @@ +../thrash/2-recovery-overrides \ No newline at end of file diff --git a/ceph/qa/suites/rados/thrash-erasure-code/recovery-overrides b/ceph/qa/suites/rados/thrash-erasure-code/recovery-overrides new file mode 120000 index 000000000..1957f2c42 --- /dev/null +++ b/ceph/qa/suites/rados/thrash-erasure-code/recovery-overrides @@ -0,0 +1 @@ +../thrash/2-recovery-overrides \ No newline at end of file diff --git a/ceph/qa/suites/rados/thrash/2-recovery-overrides/$ b/ceph/qa/suites/rados/thrash/2-recovery-overrides/$ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/rados/thrash/2-recovery-overrides/default.yaml b/ceph/qa/suites/rados/thrash/2-recovery-overrides/default.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/rados/thrash/2-recovery-overrides/more-active-recovery.yaml b/ceph/qa/suites/rados/thrash/2-recovery-overrides/more-active-recovery.yaml new file mode 120000 index 000000000..3aa7ce922 --- /dev/null +++ b/ceph/qa/suites/rados/thrash/2-recovery-overrides/more-active-recovery.yaml @@ -0,0 +1 @@ +../../../../overrides/more-active-recovery.yaml \ No newline at end of file diff --git a/ceph/qa/suites/rados/thrash/workloads/rados_api_tests.yaml b/ceph/qa/suites/rados/thrash/workloads/rados_api_tests.yaml index 23c705d1e..c5efb0cc0 100644 --- a/ceph/qa/suites/rados/thrash/workloads/rados_api_tests.yaml +++ b/ceph/qa/suites/rados/thrash/workloads/rados_api_tests.yaml @@ -3,6 +3,7 @@ overrides: log-whitelist: - reached quota - \(POOL_APP_NOT_ENABLED\) + - \(PG_AVAILABILITY\) crush_tunables: hammer conf: client: diff --git a/ceph/qa/suites/rados/verify/tasks/rados_api_tests.yaml b/ceph/qa/suites/rados/verify/tasks/rados_api_tests.yaml index 05b843ebd..4a06055b5 100644 --- a/ceph/qa/suites/rados/verify/tasks/rados_api_tests.yaml +++ b/ceph/qa/suites/rados/verify/tasks/rados_api_tests.yaml @@ -9,6 +9,7 @@ overrides: - \(REQUEST_SLOW\) - \(CACHE_POOL_NEAR_FULL\) - \(POOL_APP_NOT_ENABLED\) + - \(PG_AVAILABILITY\) conf: client: debug ms: 1 diff --git a/ceph/qa/suites/rbd/basic/msgr-failures/many.yaml b/ceph/qa/suites/rbd/basic/msgr-failures/many.yaml deleted file mode 100644 index 86f8dde8a..000000000 --- a/ceph/qa/suites/rbd/basic/msgr-failures/many.yaml +++ /dev/null @@ -1,5 +0,0 @@ -overrides: - ceph: - conf: - global: - ms inject socket failures: 500 diff --git a/ceph/qa/suites/rbd/openstack/base/install.yaml b/ceph/qa/suites/rbd/openstack/base/install.yaml index 90f80dcf2..740d32c00 100644 --- a/ceph/qa/suites/rbd/openstack/base/install.yaml +++ b/ceph/qa/suites/rbd/openstack/base/install.yaml @@ -4,4 +4,4 @@ tasks: overrides: ceph: log-whitelist: - - (POOL_APP_NOT_ENABLED) + - \(POOL_APP_NOT_ENABLED\) diff --git a/ceph/qa/suites/rbd/thrash/thrashers/cache.yaml b/ceph/qa/suites/rbd/thrash/thrashers/cache.yaml index e723e0929..249564841 100644 --- a/ceph/qa/suites/rbd/thrash/thrashers/cache.yaml +++ b/ceph/qa/suites/rbd/thrash/thrashers/cache.yaml @@ -4,8 +4,8 @@ overrides: - but it is still running - objects unfound and apparently lost - overall HEALTH_ - - (CACHE_POOL_NEAR_FULL) - - (CACHE_POOL_NO_HIT_SET) + - \(CACHE_POOL_NEAR_FULL\) + - \(CACHE_POOL_NO_HIT_SET\) tasks: - exec: client.0: diff --git a/ceph/qa/suites/smoke/basic/tasks/rados_api_tests.yaml b/ceph/qa/suites/smoke/basic/tasks/rados_api_tests.yaml index d17f60db1..df28ed6a2 100644 --- a/ceph/qa/suites/smoke/basic/tasks/rados_api_tests.yaml +++ b/ceph/qa/suites/smoke/basic/tasks/rados_api_tests.yaml @@ -6,7 +6,7 @@ tasks: - reached quota - but it is still running - objects unfound and apparently lost - - (POOL_APP_NOT_ENABLED) + - \(POOL_APP_NOT_ENABLED\) - thrashosds: chance_pgnum_grow: 2 chance_pgpnum_fix: 1 diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/% b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/% new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/0-cluster/+ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/0-cluster/+ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/0-cluster/openstack.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/0-cluster/openstack.yaml new file mode 100644 index 000000000..b0f3b9b4d --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/0-cluster/openstack.yaml @@ -0,0 +1,4 @@ +openstack: + - volumes: # attached to each instance + count: 4 + size: 30 # GB diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/0-cluster/start.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/0-cluster/start.yaml new file mode 100644 index 000000000..dff144f05 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/0-cluster/start.yaml @@ -0,0 +1,15 @@ +roles: +- - mon.a + - mon.b + - mon.c + - osd.0 + - osd.1 + - osd.2 + - mgr.x +- - client.0 +overrides: + ceph: + log-whitelist: + - failed to encode map + - CACHE_POOL_NO_HIT_SET + fs: xfs diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/1-install/luminous-client-x.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/1-install/luminous-client-x.yaml new file mode 100644 index 000000000..3e190f0d5 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/1-install/luminous-client-x.yaml @@ -0,0 +1,11 @@ +tasks: +- install: + branch: luminous + exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev'] +- print: "**** done install luminous" +upgrade_workload: + sequential: + - install.upgrade: + exclude_packages: ['ceph-test', 'ceph-test-dbg','libcephfs1'] + client.0: + - print: "**** done install.upgrade to -x on client.0" diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/2-workload/rbd_api_tests.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/2-workload/rbd_api_tests.yaml new file mode 100644 index 000000000..386c5e3dc --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/2-workload/rbd_api_tests.yaml @@ -0,0 +1,21 @@ +tasks: +- exec: + client.0: + - "cp $(which ceph_test_librbd_api) $TESTDIR/ceph_test_librbd_api" +- sequential: + - upgrade_workload +- ceph: +- print: "**** done ceph" +- exec: + client.0: + - "cp --force $TESTDIR/ceph_test_librbd_api $(which ceph_test_librbd_api)" + - "rm -rf $TESTDIR/ceph_test_librbd_api" +- print: "**** done reverting to luminous ceph_test_librbd_api" +- workunit: + branch: luminous + clients: + client.0: + - rbd/test_librbd_api.sh + env: + RBD_FEATURES: "61" +- print: "**** done rbd/test_librbd_api.sh" diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/2-workload/rbd_cli_import_export.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/2-workload/rbd_cli_import_export.yaml new file mode 100644 index 000000000..1467a0487 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/2-workload/rbd_cli_import_export.yaml @@ -0,0 +1,11 @@ +tasks: +- sequential: + - upgrade_workload +- ceph: +- print: "**** done ceph" +- workunit: + branch: luminous + clients: + client.0: + - rbd/import_export.sh +- print: "**** done rbd/import_export.sh" diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/supported/centos_7.4.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/supported/centos_7.4.yaml new file mode 120000 index 000000000..dcc001742 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/supported/centos_7.4.yaml @@ -0,0 +1 @@ +../../../../../../distros/all/centos_7.4.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/supported/rhel_7.5.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/supported/rhel_7.5.yaml new file mode 120000 index 000000000..72133969a --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/supported/rhel_7.5.yaml @@ -0,0 +1 @@ +../../../../../../distros/all/rhel_7.5.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/supported/ubuntu_16.04.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/supported/ubuntu_16.04.yaml new file mode 120000 index 000000000..9dc1ea992 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/supported/ubuntu_16.04.yaml @@ -0,0 +1 @@ +../../../../../../distros/all/ubuntu_16.04.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/supported/ubuntu_18.04.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/supported/ubuntu_18.04.yaml new file mode 120000 index 000000000..886e87fa2 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/basic/supported/ubuntu_18.04.yaml @@ -0,0 +1 @@ +../../../../../../distros/all/ubuntu_18.04.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/% b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/% new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/0-cluster/+ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/0-cluster/+ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/0-cluster/openstack.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/0-cluster/openstack.yaml new file mode 100644 index 000000000..b0f3b9b4d --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/0-cluster/openstack.yaml @@ -0,0 +1,4 @@ +openstack: + - volumes: # attached to each instance + count: 4 + size: 30 # GB diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/0-cluster/start.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/0-cluster/start.yaml new file mode 100644 index 000000000..699811f65 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/0-cluster/start.yaml @@ -0,0 +1,15 @@ +roles: +- - mon.a + - mon.b + - mon.c + - osd.0 + - osd.1 + - osd.2 + - client.0 + - mgr.x +- - client.1 +overrides: + ceph: + log-whitelist: + - failed to encode map + fs: xfs diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/1-install/luminous-client-x.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/1-install/luminous-client-x.yaml new file mode 100644 index 000000000..4a4553546 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/1-install/luminous-client-x.yaml @@ -0,0 +1,11 @@ +tasks: +- install: + branch: luminous + exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev'] +- print: "**** done install luminous" +- install.upgrade: + exclude_packages: ['ceph-test', 'ceph-test-dbg','libcephfs1'] + client.1: +- print: "**** done install.upgrade to -x on client.0" +- ceph: +- print: "**** done ceph task" diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/2-features/defaults.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/2-features/defaults.yaml new file mode 100644 index 000000000..dff6623ad --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/2-features/defaults.yaml @@ -0,0 +1,6 @@ +overrides: + ceph: + conf: + client: + rbd default features: 61 + diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/2-features/layering.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/2-features/layering.yaml new file mode 100644 index 000000000..5613d0155 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/2-features/layering.yaml @@ -0,0 +1,6 @@ +overrides: + ceph: + conf: + client: + rbd default features: 1 + diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/3-workload/rbd_notification_tests.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/3-workload/rbd_notification_tests.yaml new file mode 100644 index 000000000..17d2c17db --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/3-workload/rbd_notification_tests.yaml @@ -0,0 +1,21 @@ +tasks: +- workunit: + branch: luminous + clients: + client.0: + - rbd/notify_master.sh + client.1: + - rbd/notify_slave.sh + env: + RBD_FEATURES: "61" +- print: "**** done rbd: old librbd -> new librbd" +- workunit: + branch: luminous + clients: + client.0: + - rbd/notify_slave.sh + client.1: + - rbd/notify_master.sh + env: + RBD_FEATURES: "61" +- print: "**** done rbd: new librbd -> old librbd" diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/supported/centos_7.4.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/supported/centos_7.4.yaml new file mode 120000 index 000000000..dcc001742 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/supported/centos_7.4.yaml @@ -0,0 +1 @@ +../../../../../../distros/all/centos_7.4.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/supported/rhel_7.5.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/supported/rhel_7.5.yaml new file mode 120000 index 000000000..72133969a --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/supported/rhel_7.5.yaml @@ -0,0 +1 @@ +../../../../../../distros/all/rhel_7.5.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/supported/ubuntu_16.04.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/supported/ubuntu_16.04.yaml new file mode 120000 index 000000000..9dc1ea992 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/supported/ubuntu_16.04.yaml @@ -0,0 +1 @@ +../../../../../../distros/all/ubuntu_16.04.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/supported/ubuntu_18.04.yaml b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/supported/ubuntu_18.04.yaml new file mode 120000 index 000000000..886e87fa2 --- /dev/null +++ b/ceph/qa/suites/upgrade/client-upgrade-luminous/luminous-client-x/rbd/supported/ubuntu_18.04.yaml @@ -0,0 +1 @@ +../../../../../../distros/all/ubuntu_18.04.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/jewel-x/parallel/1-jewel-install/jewel.yaml b/ceph/qa/suites/upgrade/jewel-x/parallel/1-jewel-install/jewel.yaml index c64b2cded..48f1ce328 100644 --- a/ceph/qa/suites/upgrade/jewel-x/parallel/1-jewel-install/jewel.yaml +++ b/ceph/qa/suites/upgrade/jewel-x/parallel/1-jewel-install/jewel.yaml @@ -42,7 +42,6 @@ tasks: - \(PG_ - Monitor daemon marked osd - Behind on trimming - - is unresponsive conf: global: mon warn on pool no app: false diff --git a/ceph/qa/suites/upgrade/kraken-x/parallel/0-cluster/start.yaml b/ceph/qa/suites/upgrade/kraken-x/parallel/0-cluster/start.yaml index f5a883a39..4d10158b7 100644 --- a/ceph/qa/suites/upgrade/kraken-x/parallel/0-cluster/start.yaml +++ b/ceph/qa/suites/upgrade/kraken-x/parallel/0-cluster/start.yaml @@ -25,7 +25,7 @@ overrides: - scrub mismatch - ScrubResult - wrongly marked - - (POOL_APP_NOT_ENABLED) + - \(POOL_APP_NOT_ENABLED\) - overall HEALTH_ conf: global: diff --git a/ceph/qa/suites/upgrade/luminous-p2p/% b/ceph/qa/suites/upgrade/luminous-p2p/% new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/luminous-x/point-to-point-x/point-to-point-upgrade.yaml b/ceph/qa/suites/upgrade/luminous-p2p/point-to-point-upgrade.yaml similarity index 62% rename from ceph/qa/suites/upgrade/luminous-x/point-to-point-x/point-to-point-upgrade.yaml rename to ceph/qa/suites/upgrade/luminous-p2p/point-to-point-upgrade.yaml index 0a4ed9ac8..f660e2a2f 100644 --- a/ceph/qa/suites/upgrade/luminous-x/point-to-point-x/point-to-point-upgrade.yaml +++ b/ceph/qa/suites/upgrade/luminous-p2p/point-to-point-upgrade.yaml @@ -5,9 +5,9 @@ meta: Use xfs beneath the osds. install ceph/luminous v12.2.2 point version run workload and upgrade-sequence in parallel - install ceph/luminous latest version + install ceph/luminous v12.2.5 point version run workload and upgrade-sequence in parallel - install ceph/-x version (luminous or master/mimic) + install ceph/luminous latest version run workload and upgrade-sequence in parallel overrides: ceph: @@ -16,6 +16,18 @@ overrides: - scrub - osd_map_max_advance - wrongly marked + - FS_DEGRADED + - POOL_APP_NOT_ENABLED + - CACHE_POOL_NO_HIT_SET + - POOL_FULL + - SMALLER_PG + - pool\(s\) full + - OSD_DOWN + - missing hit_sets + - CACHE_POOL_NEAR_FULL + - PG_AVAILABILITY + - PG_DEGRADED + - application not enabled fs: xfs conf: mon: @@ -61,44 +73,33 @@ tasks: - print: "**** done ceph xfs" - sequential: - workload -- print: "**** done workload" +- print: "**** done workload v12.2.2" + +#### upgrade to v12.2.5 - install.upgrade: #exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev'] mon.a: - branch: luminous + tag: v12.2.5 mon.b: - branch: luminous + tag: v12.2.5 # Note that client.a IS NOT upgraded at this point - parallel: - workload_luminous - upgrade-sequence_luminous -- print: "**** done parallel luminous branch" +- print: "**** done parallel luminous v12.2.5" +#### upgrade to latest luminous - install.upgrade: #exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev'] - client.1: - branch: luminous -- print: "**** done branch: luminous install.upgrade on client.1" -- install.upgrade: mon.a: + branch: luminous mon.b: -- print: "**** done branch: -x install.upgrade on mon.a and mon.b" + branch: luminous + # Note that client.a IS NOT upgraded at this point - parallel: - - workload_x - - upgrade-sequence_x -- print: "**** done parallel -x branch" -- exec: - osd.0: - - ceph osd set-require-min-compat-client luminous -# Run librados tests on the -x upgraded cluster -- install.upgrade: - client.1: -- workunit: - branch: luminous - clients: - client.1: - - rados/test.sh - - cls -- print: "**** done final test on -x cluster" + - workload_luminous + - upgrade-sequence_luminous +- print: "**** done parallel luminous branch" + ####################### workload: sequential: @@ -160,68 +161,3 @@ upgrade-sequence_luminous: - sleep: duration: 60 - print: "**** done ceph.restart all luminous branch mds/osd/mon" -workload_x: - sequential: - - workunit: - branch: luminous - clients: - client.1: - - rados/test-upgrade-to-mimic.sh - - cls - - print: "**** done rados/test-upgrade-to-mimic.sh & cls workload_x NOT upgraded client" - - workunit: - branch: luminous - clients: - client.0: - - rados/test-upgrade-to-mimic.sh - - cls - - print: "**** done rados/test.sh & cls workload_x upgraded client" - - rgw: [client.1] - - print: "**** done rgw workload_x" - - s3tests: - client.1: - force-branch: ceph-luminous - rgw_server: client.1 - scan_for_encryption_keys: false - - print: "**** done s3tests workload_x" -upgrade-sequence_x: - sequential: - - ceph.restart: [mds.a] - - sleep: - duration: 60 - - ceph.restart: [mon.a] - - sleep: - duration: 60 - - ceph.restart: [mon.b] - - sleep: - duration: 60 - - ceph.restart: [mon.c] - - sleep: - duration: 60 - - ceph.restart: [osd.0] - - sleep: - duration: 30 - - ceph.restart: [osd.1] - - sleep: - duration: 30 - - ceph.restart: [osd.2] - - sleep: - duration: 30 - - ceph.restart: [osd.3] - - sleep: - duration: 30 - - ceph.restart: [osd.4] - - sleep: - duration: 30 - - ceph.restart: - daemons: [osd.5] - wait-for-healthy: false - wait-for-up-osds: true - - ceph.restart: - daemons: [mgr.x] - wait-for-healthy: false - - exec: - osd.0: - - ceph osd require-osd-release luminous - - ceph.healthy: - - print: "**** done ceph.restart all -x branch mds/osd/mon" diff --git a/ceph/qa/suites/upgrade/luminous-p2p/supported b/ceph/qa/suites/upgrade/luminous-p2p/supported new file mode 120000 index 000000000..dd0d7f1d5 --- /dev/null +++ b/ceph/qa/suites/upgrade/luminous-p2p/supported @@ -0,0 +1 @@ +../../../distros/supported/ \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/start.yaml b/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/start.yaml index 3684b1e0a..f8732212d 100644 --- a/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/start.yaml +++ b/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/start.yaml @@ -25,7 +25,7 @@ overrides: - scrub mismatch - ScrubResult - wrongly marked - - (POOL_APP_NOT_ENABLED) + - \(POOL_APP_NOT_ENABLED\) - overall HEALTH_ conf: global: diff --git a/ceph/qa/suites/upgrade/luminous-x/point-to-point-x/distros/centos_latest.yaml b/ceph/qa/suites/upgrade/luminous-x/point-to-point-x/distros/centos_latest.yaml deleted file mode 120000 index b5973b952..000000000 --- a/ceph/qa/suites/upgrade/luminous-x/point-to-point-x/distros/centos_latest.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../../../distros/supported/centos_latest.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/luminous-x/point-to-point-x/distros/ubuntu_latest.yaml b/ceph/qa/suites/upgrade/luminous-x/point-to-point-x/distros/ubuntu_latest.yaml deleted file mode 120000 index cc5b15bcc..000000000 --- a/ceph/qa/suites/upgrade/luminous-x/point-to-point-x/distros/ubuntu_latest.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../../../distros/supported/ubuntu_latest.yaml \ No newline at end of file diff --git a/ceph/qa/tasks/ceph.py b/ceph/qa/tasks/ceph.py index 87cbbc1ff..f61047d98 100644 --- a/ceph/qa/tasks/ceph.py +++ b/ceph/qa/tasks/ceph.py @@ -1059,6 +1059,8 @@ def osd_scrub_pgs(ctx, config): # allow this to fail; in certain cases the OSD might not be up # at this point. we will catch all pgs below. try: + manager.raw_cluster_cmd('tell', 'osd.' + id_, 'config', 'set', + 'osd_debug_deep_scrub_sleep', '0'); manager.raw_cluster_cmd('osd', 'deep-scrub', id_) except run.CommandFailedError: pass diff --git a/ceph/qa/tasks/ceph_deploy.py b/ceph/qa/tasks/ceph_deploy.py index 7d65ca1c7..3d4c7c8d3 100644 --- a/ceph/qa/tasks/ceph_deploy.py +++ b/ceph/qa/tasks/ceph_deploy.py @@ -483,6 +483,19 @@ def build_ceph_cluster(ctx, config): elif not config.get('only_mon'): raise RuntimeError( "The cluster is NOT operational due to insufficient OSDs") + # create rbd pool + ceph_admin.run( + args=[ + 'sudo', 'ceph', '--cluster', 'ceph', + 'osd', 'pool', 'create', 'rbd', '128', '128'], + check_status=False) + ceph_admin.run( + args=[ + 'sudo', 'ceph', '--cluster', 'ceph', + 'osd', 'pool', 'application', 'enable', + 'rbd', 'rbd', '--yes-i-really-mean-it' + ], + check_status=False) yield except Exception: diff --git a/ceph/qa/tasks/ceph_manager.py b/ceph/qa/tasks/ceph_manager.py index edbb2ae3e..969cd6e23 100644 --- a/ceph/qa/tasks/ceph_manager.py +++ b/ceph/qa/tasks/ceph_manager.py @@ -1437,7 +1437,7 @@ class CephManager: # both osd_mon_report_interval_min and mgr_stats_period are 5 seconds # by default, and take the faulty injection in ms into consideration, # 12 seconds are more than enough - delays = [1, 1, 2, 3, 5, 8, 13] + delays = [1, 1, 2, 3, 5, 8, 13, 0] @wraps(func) def wrapper(self, *args, **kwargs): exc = None diff --git a/ceph/qa/tasks/cephfs/filesystem.py b/ceph/qa/tasks/cephfs/filesystem.py index 6cc1ea558..b39504419 100644 --- a/ceph/qa/tasks/cephfs/filesystem.py +++ b/ceph/qa/tasks/cephfs/filesystem.py @@ -714,7 +714,17 @@ class Filesystem(MDSCluster): return result - def get_rank_names(self): + def get_rank(self, rank=0, status=None): + if status is None: + status = self.getinfo() + return status.get_rank(self.id, rank) + + def get_ranks(self, status=None): + if status is None: + status = self.getinfo() + return status.get_ranks(self.id) + + def get_rank_names(self, status=None): """ Return MDS daemon names of those daemons holding a rank, sorted by rank. This includes e.g. up:replay/reconnect @@ -838,6 +848,10 @@ class Filesystem(MDSCluster): return self.json_asok(command, 'mds', mds_id) + def rank_asok(self, command, rank=0): + info = self.get_rank(rank=rank) + return self.json_asok(command, 'mds', info['name']) + def read_cache(self, path, depth=None): cmd = ["dump", "tree", path] if depth is not None: diff --git a/ceph/qa/tasks/cephfs/fuse_mount.py b/ceph/qa/tasks/cephfs/fuse_mount.py index 8d8410c69..b121680b0 100644 --- a/ceph/qa/tasks/cephfs/fuse_mount.py +++ b/ceph/qa/tasks/cephfs/fuse_mount.py @@ -261,7 +261,7 @@ class FuseMount(CephFSMount): assert not self.is_mounted() self._fuse_conn = None - def umount_wait(self, force=False, require_clean=False): + def umount_wait(self, force=False, require_clean=False, timeout=900): """ :param force: Complete cleanly even if the MDS is offline """ @@ -282,7 +282,7 @@ class FuseMount(CephFSMount): try: if self.fuse_daemon: # Permit a timeout, so that we do not block forever - run.wait([self.fuse_daemon], 900) + run.wait([self.fuse_daemon], timeout) except MaxWhileTries: log.error("process failed to terminate after unmount. This probably" "indicates a bug within ceph-fuse.") @@ -407,9 +407,15 @@ print find_socket("{client_name}") """ Look up the CephFS client ID for this mount """ - return self.admin_socket(['mds_sessions'])['id'] + def get_client_pid(self): + """ + return pid of ceph-fuse process + """ + status = self.admin_socket(['status']) + return status['metadata']['pid'] + def get_osd_epoch(self): """ Return 2-tuple of osd_epoch, osd_epoch_barrier diff --git a/ceph/qa/tasks/cephfs/kernel_mount.py b/ceph/qa/tasks/cephfs/kernel_mount.py index bfa1ac679..d237f65fb 100644 --- a/ceph/qa/tasks/cephfs/kernel_mount.py +++ b/ceph/qa/tasks/cephfs/kernel_mount.py @@ -124,7 +124,7 @@ class KernelMount(CephFSMount): def cleanup(self): pass - def umount_wait(self, force=False, require_clean=False): + def umount_wait(self, force=False, require_clean=False, timeout=900): """ Unlike the fuse client, the kernel client's umount is immediate """ diff --git a/ceph/qa/tasks/cephfs/test_client_recovery.py b/ceph/qa/tasks/cephfs/test_client_recovery.py index fd58c1427..829ca3d5c 100644 --- a/ceph/qa/tasks/cephfs/test_client_recovery.py +++ b/ceph/qa/tasks/cephfs/test_client_recovery.py @@ -11,8 +11,10 @@ import re import os from teuthology.orchestra.run import CommandFailedError, ConnectionLostError +from tasks.cephfs.fuse_mount import FuseMount from tasks.cephfs.cephfs_test_case import CephFSTestCase from teuthology.packaging import get_package_version +from unittest import SkipTest log = logging.getLogger(__name__) @@ -472,3 +474,42 @@ class TestClientRecovery(CephFSTestCase): self.mount_b.mount() self.mount_b.wait_until_mounted() self.mount_b.run_shell(["ls", "subdir/childfile"]) + + def test_stale_renew(self): + if not isinstance(self.mount_a, FuseMount): + raise SkipTest("Require FUSE client to handle signal STOP/CONT") + + self.mount_a.run_shell(["mkdir", "testdir"]) + self.mount_a.run_shell(["touch", "testdir/file1"]) + # populate readdir cache + self.mount_a.run_shell(["ls", "testdir"]) + self.mount_b.run_shell(["ls", "testdir"]) + + # check if readdir cache is effective + initial_readdirs = self.fs.mds_asok(['perf', 'dump', 'mds_server', 'req_readdir_latency']) + self.mount_b.run_shell(["ls", "testdir"]) + current_readdirs = self.fs.mds_asok(['perf', 'dump', 'mds_server', 'req_readdir_latency']) + self.assertEqual(current_readdirs, initial_readdirs); + + mount_b_gid = self.mount_b.get_global_id() + mount_b_pid = self.mount_b.get_client_pid() + # stop ceph-fuse process of mount_b + self.mount_b.client_remote.run(args=["sudo", "kill", "-STOP", mount_b_pid]) + + self.assert_session_state(mount_b_gid, "open") + time.sleep(self.mds_session_timeout * 1.5) # Long enough for MDS to consider session stale + self.assert_session_state(mount_b_gid, "stale") + + self.mount_a.run_shell(["touch", "testdir/file2"]) + + # resume ceph-fuse process of mount_b + self.mount_b.client_remote.run(args=["sudo", "kill", "-CONT", mount_b_pid]) + # Is the new file visible from mount_b? (caps become invalid after session stale) + self.mount_b.run_shell(["ls", "testdir/file2"]) + + def test_unmount_for_evicted_client(self): + """Test if client hangs on unmount after evicting the client.""" + mount_a_client_id = self.mount_a.get_global_id() + self.fs.mds_asok(['session', 'evict', "%s" % mount_a_client_id]) + + self.mount_a.umount_wait(require_clean=True, timeout=30) diff --git a/ceph/qa/tasks/cephfs/test_exports.py b/ceph/qa/tasks/cephfs/test_exports.py index 913999db7..2c6231359 100644 --- a/ceph/qa/tasks/cephfs/test_exports.py +++ b/ceph/qa/tasks/cephfs/test_exports.py @@ -7,6 +7,7 @@ log = logging.getLogger(__name__) class TestExports(CephFSTestCase): MDSS_REQUIRED = 2 + CLIENTS_REQUIRED = 2 def _wait_subtrees(self, status, rank, test): timeout = 30 @@ -105,3 +106,42 @@ class TestExports(CephFSTestCase): self._wait_subtrees(status, 0, [('/1', 0), ('/1/4/5', 1), ('/1/2/3', 2), ('/a', 1), ('/aa/bb', 0)]) self.mount_a.run_shell(["mv", "aa", "a/b/"]) self._wait_subtrees(status, 0, [('/1', 0), ('/1/4/5', 1), ('/1/2/3', 2), ('/a', 1), ('/a/b/aa/bb', 0)]) + + def test_session_race(self): + """ + Test session creation race. + + See: https://tracker.ceph.com/issues/24072#change-113056 + """ + + self.fs.set_max_mds(2) + self.fs.wait_for_daemons() + + status = self.fs.status() + rank1 = self.fs.get_rank(rank=1, status=status) + name1 = 'mds.'+rank1['name'] + + # Create a directory that is pre-exported to rank 1 + self.mount_a.run_shell(["mkdir", "-p", "a/aa"]) + self.mount_a.setfattr("a", "ceph.dir.pin", "1") + self._wait_subtrees(status, 1, [('/a', 1)]) + + # Now set the mds config to allow the race + self.fs.rank_asok(["config", "set", "mds_inject_migrator_session_race", "true"], rank=1) + + # Now create another directory and try to export it + self.mount_b.run_shell(["mkdir", "-p", "b/bb"]) + self.mount_b.setfattr("b", "ceph.dir.pin", "1") + + time.sleep(5) + + # Now turn off the race so that it doesn't wait again + self.fs.rank_asok(["config", "set", "mds_inject_migrator_session_race", "false"], rank=1) + + # Now try to create a session with rank 1 by accessing a dir known to + # be there, if buggy, this should cause the rank 1 to crash: + self.mount_b.run_shell(["ls", "a"]) + + # Check if rank1 changed (standby tookover?) + new_rank1 = self.fs.get_rank(rank=1) + self.assertEqual(rank1['gid'], new_rank1['gid']) diff --git a/ceph/qa/tasks/cephfs/test_strays.py b/ceph/qa/tasks/cephfs/test_strays.py index b64f3e931..f96994e88 100644 --- a/ceph/qa/tasks/cephfs/test_strays.py +++ b/ceph/qa/tasks/cephfs/test_strays.py @@ -948,76 +948,6 @@ class TestStrays(CephFSTestCase): self.mds_cluster.mds_restart() self.fs.wait_for_daemons() - def test_purge_queue_op_rate(self): - """ - A busy purge queue is meant to aggregate operations sufficiently - that our RADOS ops to the metadata pool are not O(files). Check - that that is so. - :return: - """ - - # For low rates of deletion, the rate of metadata ops actually - # will be o(files), so to see the desired behaviour we have to give - # the system a significant quantity, i.e. an order of magnitude - # more than the number of files it will purge at one time. - - max_purge_files = 2 - - self.set_conf('mds', 'mds_bal_frag', 'false') - self.set_conf('mds', 'mds_max_purge_files', "%d" % max_purge_files) - self.fs.mds_fail_restart() - self.fs.wait_for_daemons() - - phase_1_files = 256 - phase_2_files = 512 - - self.mount_a.run_shell(["mkdir", "phase1"]) - self.mount_a.create_n_files("phase1/file", phase_1_files) - - self.mount_a.run_shell(["mkdir", "phase2"]) - self.mount_a.create_n_files("phase2/file", phase_2_files) - - def unlink_and_count_ops(path, expected_deletions): - initial_ops = self.get_stat("objecter", "op") - initial_pq_executed = self.get_stat("purge_queue", "pq_executed") - - self.mount_a.run_shell(["rm", "-rf", path]) - - self._wait_for_counter( - "purge_queue", "pq_executed", initial_pq_executed + expected_deletions - ) - - final_ops = self.get_stat("objecter", "op") - - # Calculation of the *overhead* operations, i.e. do not include - # the operations where we actually delete files. - return final_ops - initial_ops - expected_deletions - - self.fs.mds_asok(['flush', 'journal']) - phase1_ops = unlink_and_count_ops("phase1/", phase_1_files + 1) - - self.fs.mds_asok(['flush', 'journal']) - phase2_ops = unlink_and_count_ops("phase2/", phase_2_files + 1) - - log.info("Phase 1: {0}".format(phase1_ops)) - log.info("Phase 2: {0}".format(phase2_ops)) - - # The success criterion is that deleting double the number - # of files doesn't generate double the number of overhead ops - # -- this comparison is a rough approximation of that rule. - self.assertTrue(phase2_ops < phase1_ops * 1.25) - - # Finally, check that our activity did include properly quiescing - # the queue (i.e. call to Journaler::write_head in the right place), - # by restarting the MDS and checking that it doesn't try re-executing - # any of the work we did. - self.fs.mds_asok(['flush', 'journal']) # flush to ensure no strays - # hanging around - self.fs.mds_fail_restart() - self.fs.wait_for_daemons() - time.sleep(10) - self.assertEqual(self.get_stat("purge_queue", "pq_executed"), 0) - def test_replicated_delete_speed(self): """ That deletions of replicated metadata are not pathologically slow diff --git a/ceph/qa/tasks/cephfs/test_volume_client.py b/ceph/qa/tasks/cephfs/test_volume_client.py index 0876af96e..9be7fc2ff 100644 --- a/ceph/qa/tasks/cephfs/test_volume_client.py +++ b/ceph/qa/tasks/cephfs/test_volume_client.py @@ -283,21 +283,21 @@ vc.disconnect() # it has lost network, because there is nothing to tell it that is messages # are being dropped because it's identity is gone) background = self.mounts[2].write_n_mb("rogue.bin", 1, wait=False) - time.sleep(10) # Approximate check for 'stuck' as 'still running after 10s' - self.assertFalse(background.finished) + try: + background.wait() + except CommandFailedError: + # command failed with EBLACKLISTED? + if "transport endpoint shutdown" in background.stderr.getvalue(): + pass + else: + raise # After deauthorisation, the client ID should be gone (this was the only # volume it was authorised for) self.assertNotIn("client.{0}".format(guest_entity), [e['entity'] for e in self.auth_list()]) # Clean up the dead mount (ceph-fuse's behaviour here is a bit undefined) - self.mounts[2].kill() - self.mounts[2].kill_cleanup() - try: - background.wait() - except CommandFailedError: - # We killed the mount out from under you - pass + self.mounts[2].umount_wait() self._volume_client_python(self.mount_b, dedent(""" vp = VolumePath("{group_id}", "{volume_id}") @@ -343,6 +343,19 @@ vc.disconnect() vc.delete_volume(vp, data_isolated=True) vc.purge_volume(vp, data_isolated=True) vc.purge_volume(vp, data_isolated=True) + + vc.create_volume(vp, 10, namespace_isolated=False) + vc.create_volume(vp, 10, namespace_isolated=False) + vc.authorize(vp, "{guest_entity}") + vc.authorize(vp, "{guest_entity}") + vc.deauthorize(vp, "{guest_entity}") + vc.deauthorize(vp, "{guest_entity}") + vc.evict("{guest_entity}") + vc.evict("{guest_entity}") + vc.delete_volume(vp) + vc.delete_volume(vp) + vc.purge_volume(vp) + vc.purge_volume(vp) """.format( group_id=group_id, volume_id=volume_id, @@ -764,52 +777,63 @@ vc.disconnect() # auth ID belongs to, the auth ID's authorized access levels # for different volumes, versioning details, etc. expected_auth_metadata = { - u"version": 2, - u"compat_version": 1, - u"dirty": False, - u"tenant_id": u"tenant1", - u"volumes": { - u"groupid/volumeid": { - u"dirty": False, - u"access_level": u"rw", + "version": 2, + "compat_version": 1, + "dirty": False, + "tenant_id": u"tenant1", + "volumes": { + "groupid/volumeid": { + "dirty": False, + "access_level": u"rw", } } } auth_metadata = self._volume_client_python(volumeclient_mount, dedent(""" + import json vp = VolumePath("{group_id}", "{volume_id}") auth_metadata = vc._auth_metadata_get("{auth_id}") - print auth_metadata + print(json.dumps(auth_metadata)) """.format( group_id=group_id, volume_id=volume_id, auth_id=guestclient_1["auth_id"], ))) + auth_metadata = json.loads(auth_metadata) - self.assertItemsEqual(str(expected_auth_metadata), auth_metadata) + self.assertGreaterEqual(auth_metadata["version"], expected_auth_metadata["version"]) + del expected_auth_metadata["version"] + del auth_metadata["version"] + self.assertEqual(expected_auth_metadata, auth_metadata) # Verify that the volume metadata file stores info about auth IDs # and their access levels to the volume, versioning details, etc. expected_vol_metadata = { - u"version": 2, - u"compat_version": 1, - u"auths": { - u"guest": { - u"dirty": False, - u"access_level": u"rw" + "version": 2, + "compat_version": 1, + "auths": { + "guest": { + "dirty": False, + "access_level": u"rw" } } } vol_metadata = self._volume_client_python(volumeclient_mount, dedent(""" + import json vp = VolumePath("{group_id}", "{volume_id}") volume_metadata = vc._volume_metadata_get(vp) - print volume_metadata + print(json.dumps(volume_metadata)) """.format( group_id=group_id, volume_id=volume_id, ))) - self.assertItemsEqual(str(expected_vol_metadata), vol_metadata) + vol_metadata = json.loads(vol_metadata) + + self.assertGreaterEqual(vol_metadata["version"], expected_vol_metadata["version"]) + del expected_vol_metadata["version"] + del vol_metadata["version"] + self.assertEqual(expected_vol_metadata, vol_metadata) # Cannot authorize 'guestclient_2' to access the volume. # It uses auth ID 'guest', which has already been used by a @@ -1014,3 +1038,43 @@ vc.disconnect() # Mount the volume in the guest using the auth ID to assert that the # auth caps are valid guest_mount.mount(mount_path=mount_path) + + def test_volume_without_namespace_isolation(self): + """ + That volume client can create volumes that do not have separate RADOS + namespace layouts. + """ + vc_mount = self.mounts[1] + vc_mount.umount_wait() + + # Configure vc_mount as the handle for driving volumeclient + self._configure_vc_auth(vc_mount, "manila") + + # Create a volume + volume_prefix = "/myprefix" + group_id = "grpid" + volume_id = "volid" + mount_path = self._volume_client_python(vc_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + create_result = vc.create_volume(vp, 1024*1024*10, namespace_isolated=False) + print create_result['mount_path'] + """.format( + group_id=group_id, + volume_id=volume_id + )), volume_prefix) + + # The CephFS volume should be created + self.mounts[0].stat(os.path.join("myprefix", group_id, volume_id)) + vol_namespace = self.mounts[0].getfattr( + os.path.join("myprefix", group_id, volume_id), + "ceph.dir.layout.pool_namespace") + assert not vol_namespace + + self._volume_client_python(vc_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + vc.delete_volume(vp) + vc.purge_volume(vp) + """.format( + group_id=group_id, + volume_id=volume_id, + )), volume_prefix) diff --git a/ceph/qa/tasks/filestore_idempotent.py b/ceph/qa/tasks/filestore_idempotent.py index 4e2a228fc..35c1a8002 100644 --- a/ceph/qa/tasks/filestore_idempotent.py +++ b/ceph/qa/tasks/filestore_idempotent.py @@ -38,7 +38,9 @@ def task(ctx, config): dir = '%s/ceph.data/test.%s' % (testdir, client) - seed = str(int(random.uniform(1,100))) + seed = int(random.uniform(1,100)) + start = 800 + random.randint(800,1200) + end = start + 150 try: log.info('creating a working dir') @@ -61,7 +63,7 @@ def task(ctx, config): args=[ 'cd', dir, run.Raw('&&'), - './run_seed_to_range.sh', seed, '50', '300', + './run_seed_to_range.sh', str(seed), str(start), str(end), ], wait=False, check_status=False) diff --git a/ceph/qa/tasks/osd_max_pg_per_osd.py b/ceph/qa/tasks/osd_max_pg_per_osd.py index b4e2aa4de..03ea218f5 100644 --- a/ceph/qa/tasks/osd_max_pg_per_osd.py +++ b/ceph/qa/tasks/osd_max_pg_per_osd.py @@ -34,8 +34,9 @@ def test_create_from_mon(ctx, config): manager = ctx.managers['ceph'] log.info('1. creating pool.a') pool_a = manager.create_pool_with_unique_name(pg_num) - manager.wait_for_clean() - assert manager.get_num_active_clean() == pg_num + pg_states = manager.wait_till_pg_convergence(300) + pg_created = pg_num_in_all_states(pg_states, 'active', 'clean') + assert pg_created == pg_num log.info('2. creating pool.b') pool_b = manager.create_pool_with_unique_name(pg_num) @@ -81,8 +82,9 @@ def test_create_from_peer(ctx, config): manager = ctx.managers['ceph'] log.info('1. creating pool.a') pool_a = manager.create_pool_with_unique_name(pg_num) - manager.wait_for_clean() - assert manager.get_num_active_clean() == pg_num + pg_states = manager.wait_till_pg_convergence(300) + pg_created = pg_num_in_all_states(pg_states, 'active', 'clean') + assert pg_created == pg_num log.info('2. creating pool.b') while True: diff --git a/ceph/qa/tasks/radosgw_admin_rest.py b/ceph/qa/tasks/radosgw_admin_rest.py index 7bd72d195..99a742f13 100644 --- a/ceph/qa/tasks/radosgw_admin_rest.py +++ b/ceph/qa/tasks/radosgw_admin_rest.py @@ -254,6 +254,7 @@ def task(ctx, config): (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1}) assert ret == 200 assert out['suspended'] + assert out['email'] == email # TESTCASE 're-enable','user','enable','suspended user','succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : 'false'}) diff --git a/ceph/qa/workunits/ceph-tests/ceph-admin-commands.sh b/ceph/qa/workunits/ceph-tests/ceph-admin-commands.sh index 4d850c391..30e74cce5 100755 --- a/ceph/qa/workunits/ceph-tests/ceph-admin-commands.sh +++ b/ceph/qa/workunits/ceph-tests/ceph-admin-commands.sh @@ -5,7 +5,6 @@ ceph -s #list pools rados lspools #lisr rbd images -ceph osd pool create rbd 128 128 rbd ls #check that the monitors work ceph osd set nodown diff --git a/ceph/qa/workunits/fs/misc/rstats.sh b/ceph/qa/workunits/fs/misc/rstats.sh new file mode 100755 index 000000000..4c32edb23 --- /dev/null +++ b/ceph/qa/workunits/fs/misc/rstats.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +set -x + +timeout=30 +old_value="" +new_value="" + +wait_until_changed() { + name=$1 + wait=0 + while [ $wait -lt $timeout ]; do + new_value=`getfattr --only-value -n ceph.dir.$name .` + [ $new_value == $old_value ] || return 0 + sleep 1 + wait=$(($wait + 1)) + done + return 1 +} + +check_rctime() { + old_sec=$(echo $old_value | cut -d. -f1) + old_nsec=$(echo $old_value | cut -d. -f2) + new_sec=$(echo $new_value | cut -d. -f1) + new_nsec=$(echo $new_value | cut -d. -f2) + [ "$old_sec" -lt "$new_sec" ] && return 0 + [ "$old_sec" -gt "$new_sec" ] && return 1 + [ "$old_nsec" -lt "$new_nsec" ] && return 0 + return 1 +} + +# sync(3) does not make ceph-fuse flush dirty caps, because fuse kernel module +# does not notify ceph-fuse about it. Use fsync(3) instead. +fsync_path() { + cmd="import os; fd=os.open(\"$1\", os.O_RDONLY); os.fsync(fd); os.close(fd)" + python -c "$cmd" +} + +set -e + +mkdir -p rstats_testdir/d1/d2 +cd rstats_testdir + +# rfiles +old_value=`getfattr --only-value -n ceph.dir.rfiles .` +[ $old_value == 0 ] || false +touch d1/d2/f1 +wait_until_changed rfiles +[ $new_value == $(($old_value + 1)) ] || false + +# rsubdirs +old_value=`getfattr --only-value -n ceph.dir.rsubdirs .` +[ $old_value == 3 ] || false +mkdir d1/d2/d3 +wait_until_changed rsubdirs +[ $new_value == $(($old_value + 1)) ] || false + +# rbytes +old_value=`getfattr --only-value -n ceph.dir.rbytes .` +[ $old_value == 0 ] || false +echo hello > d1/d2/f2 +fsync_path d1/d2/f2 +wait_until_changed rbytes +[ $new_value == $(($old_value + 6)) ] || false + +#rctime +old_value=`getfattr --only-value -n ceph.dir.rctime .` +touch d1/d2/d3 # touch existing file +fsync_path d1/d2/d3 +wait_until_changed rctime +check_rctime + +old_value=`getfattr --only-value -n ceph.dir.rctime .` +touch d1/d2/f3 # create new file +wait_until_changed rctime +check_rctime + +cd .. +rm -rf rstats_testdir +echo OK diff --git a/ceph/qa/workunits/mon/crush_ops.sh b/ceph/qa/workunits/mon/crush_ops.sh index 348811e7c..2ae820230 100755 --- a/ceph/qa/workunits/mon/crush_ops.sh +++ b/ceph/qa/workunits/mon/crush_ops.sh @@ -202,4 +202,15 @@ ceph osd pool rm cool cool --yes-i-really-really-mean-it ceph osd pool rm cold cold --yes-i-really-really-mean-it ceph osd crush weight-set rm-compat +# weight set vs device classes vs move +ceph osd crush weight-set create-compat +ceph osd crush add-bucket fooo host +ceph osd crush move fooo root=default +ceph osd crush add-bucket barr rack +ceph osd crush move barr root=default +ceph osd crush move fooo rack=barr +ceph osd crush rm fooo +ceph osd crush rm barr +ceph osd crush weight-set rm-compat + echo OK diff --git a/ceph/qa/workunits/rados/test_envlibrados_for_rocksdb.sh b/ceph/qa/workunits/rados/test_envlibrados_for_rocksdb.sh index 94580c234..c3c802d57 100755 --- a/ceph/qa/workunits/rados/test_envlibrados_for_rocksdb.sh +++ b/ceph/qa/workunits/rados/test_envlibrados_for_rocksdb.sh @@ -71,7 +71,7 @@ git clone https://github.com/facebook/rocksdb.git --depth 1 # compile code cd rocksdb -make env_librados_test ROCKSDB_USE_LIBRADOS=1 -j8 +make env_librados_test ROCKSDB_USE_LIBRADOS=1 DISABLE_WARNING_AS_ERROR=1 -j8 echo "Copy ceph.conf" # prepare ceph.conf diff --git a/ceph/qa/workunits/rados/test_large_omap_detection.py b/ceph/qa/workunits/rados/test_large_omap_detection.py new file mode 100755 index 000000000..6a9e3f938 --- /dev/null +++ b/ceph/qa/workunits/rados/test_large_omap_detection.py @@ -0,0 +1,130 @@ +#!/usr/bin/python +# -*- mode:python -*- +# vim: ts=4 sw=4 smarttab expandtab +# +# Copyright (C) 2017 Red Hat +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# + +import json +import rados +import shlex +import subprocess +import time + +def cleanup(cluster): + cluster.delete_pool('large-omap-test-pool') + cluster.shutdown() + +def init(): + # For local testing + #cluster = rados.Rados(conffile='./ceph.conf') + cluster = rados.Rados(conffile='/etc/ceph/ceph.conf') + cluster.connect() + print("\nCluster ID: " + cluster.get_fsid()) + cluster.create_pool('large-omap-test-pool') + ioctx = cluster.open_ioctx('large-omap-test-pool') + ioctx.write_full('large-omap-test-object1', "Lorem ipsum") + op = ioctx.create_write_op() + + keys = [] + values = [] + for x in range(20001): + keys.append(str(x)) + values.append("X") + + ioctx.set_omap(op, tuple(keys), tuple(values)) + ioctx.operate_write_op(op, 'large-omap-test-object1', 0) + ioctx.release_write_op(op) + + ioctx.write_full('large-omap-test-object2', "Lorem ipsum dolor") + op = ioctx.create_write_op() + + buffer = ("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do " + "eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut " + "enim ad minim veniam, quis nostrud exercitation ullamco laboris " + "nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in " + "reprehenderit in voluptate velit esse cillum dolore eu fugiat " + "nulla pariatur. Excepteur sint occaecat cupidatat non proident, " + "sunt in culpa qui officia deserunt mollit anim id est laborum.") + + keys = [] + values = [] + for x in xrange(20000): + keys.append(str(x)) + values.append(buffer) + + ioctx.set_omap(op, tuple(keys), tuple(values)) + ioctx.operate_write_op(op, 'large-omap-test-object2', 0) + ioctx.release_write_op(op) + ioctx.close() + return cluster + +def get_deep_scrub_timestamp(pgid): + cmd = ['ceph', 'pg', 'dump', '--format=json-pretty'] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) + out = proc.communicate()[0] + for stat in json.loads(out)['pg_stats']: + if stat['pgid'] == pgid: + return stat['last_deep_scrub_stamp'] + +def wait_for_scrub(): + osds = set(); + pgs = dict(); + cmd = ['ceph', 'osd', 'map', 'large-omap-test-pool', + 'large-omap-test-object1', '--format=json-pretty'] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) + out = proc.communicate()[0] + osds.add(json.loads(out)['acting_primary']) + pgs[json.loads(out)['pgid']] = get_deep_scrub_timestamp(json.loads(out)['pgid']) + cmd = ['ceph', 'osd', 'map', 'large-omap-test-pool', + 'large-omap-test-object2', '--format=json-pretty'] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) + out = proc.communicate()[0] + osds.add(json.loads(out)['acting_primary']) + pgs[json.loads(out)['pgid']] = get_deep_scrub_timestamp(json.loads(out)['pgid']) + + for pg in pgs: + command = "ceph pg deep-scrub " + str(pg) + subprocess.check_call(shlex.split(command)) + + for pg in pgs: + RETRIES = 0 + while RETRIES < 60 and pgs[pg] == get_deep_scrub_timestamp(pg): + time.sleep(10) + RETRIES += 1 + +def check_health_output(): + RETRIES = 0 + result = 0 + while RETRIES < 6 and result != 2: + result = 0 + RETRIES += 1 + output = subprocess.check_output(["ceph", "health", "detail"]) + for line in output.splitlines(): + result += int(line.find('2 large omap objects') != -1) + time.sleep(10) + + if result != 2: + print("Error, got invalid output:") + print(output) + raise Exception + +def main(): + cluster = init() + wait_for_scrub() + check_health_output() + + cleanup(cluster) + +if __name__ == '__main__': + main() diff --git a/ceph/qa/workunits/rados/test_pool_access.sh b/ceph/qa/workunits/rados/test_pool_access.sh index 8597b7147..69604bca8 100755 --- a/ceph/qa/workunits/rados/test_pool_access.sh +++ b/ceph/qa/workunits/rados/test_pool_access.sh @@ -2,22 +2,107 @@ set -e -expect_1() +KEYRING=$(mktemp) +trap cleanup EXIT ERR HUP INT QUIT + +cleanup() { + (ceph auth del client.mon_read || true) >/dev/null 2>&1 + (ceph auth del client.mon_write || true) >/dev/null 2>&1 + + rm -f $KEYRING +} + +expect_false() { - set -x - set +e - "$@" - if [ $? == 1 ]; then return 0; else return 1; fi + set -x + if "$@"; then return 1; else return 0; fi +} + +create_pool_op() { + ID=$1 + POOL=$2 + + cat << EOF | CEPH_KEYRING="$KEYRING" python +import rados + +cluster = rados.Rados(conffile="", rados_id="${ID}") +cluster.connect() +cluster.create_pool("${POOL}") +EOF } +delete_pool_op() { + ID=$1 + POOL=$2 + + cat << EOF | CEPH_KEYRING="$KEYRING" python +import rados + +cluster = rados.Rados(conffile="", rados_id="${ID}") +cluster.connect() +cluster.delete_pool("${POOL}") +EOF +} + +create_pool_snap_op() { + ID=$1 + POOL=$2 + SNAP=$3 + + cat << EOF | CEPH_KEYRING="$KEYRING" python +import rados + +cluster = rados.Rados(conffile="", rados_id="${ID}") +cluster.connect() +ioctx = cluster.open_ioctx("${POOL}") + +ioctx.create_snap("${SNAP}") +EOF +} + +remove_pool_snap_op() { + ID=$1 + POOL=$2 + SNAP=$3 + + cat << EOF | CEPH_KEYRING="$KEYRING" python +import rados + +cluster = rados.Rados(conffile="", rados_id="${ID}") +cluster.connect() +ioctx = cluster.open_ioctx("${POOL}") + +ioctx.remove_snap("${SNAP}") +EOF +} + +test_pool_op() +{ + ceph auth get-or-create client.mon_read mon 'allow r' >> $KEYRING + ceph auth get-or-create client.mon_write mon 'allow *' >> $KEYRING + + expect_false create_pool_op mon_read pool1 + create_pool_op mon_write pool1 + + expect_false create_pool_snap_op mon_read pool1 snap1 + create_pool_snap_op mon_write pool1 snap1 + + expect_false remove_pool_snap_op mon_read pool1 snap1 + remove_pool_snap_op mon_write pool1 snap1 + + expect_false delete_pool_op mon_read pool1 + delete_pool_op mon_write pool1 +} key=`ceph auth get-or-create-key client.poolaccess1 mon 'allow r' osd 'allow *'` rados --id poolaccess1 --key $key -p rbd ls key=`ceph auth get-or-create-key client.poolaccess2 mon 'allow r' osd 'allow * pool=nopool'` -expect_1 rados --id poolaccess2 --key $key -p rbd ls +expect_false rados --id poolaccess2 --key $key -p rbd ls key=`ceph auth get-or-create-key client.poolaccess3 mon 'allow r' osd 'allow rw pool=nopool'` -expect_1 rados --id poolaccess3 --key $key -p rbd ls +expect_false rados --id poolaccess3 --key $key -p rbd ls + +test_pool_op echo OK diff --git a/ceph/qa/workunits/rbd/import_export.sh b/ceph/qa/workunits/rbd/import_export.sh index c9ecb8b62..3c85adc0f 100755 --- a/ceph/qa/workunits/rbd/import_export.sh +++ b/ceph/qa/workunits/rbd/import_export.sh @@ -79,10 +79,14 @@ if rbd help export | grep -q export-format; then dd if=/bin/dd of=${TMPDIR}/img bs=1k count=10 seek=100 rbd import $RBD_CREATE_ARGS ${TMPDIR}/img testimg rbd snap create testimg@snap + rbd image-meta set testimg key1 value1 + IMAGEMETA_BEFORE=`rbd image-meta list testimg` rbd export --export-format 2 testimg ${TMPDIR}/img_v2 rbd import --export-format 2 ${TMPDIR}/img_v2 testimg_import rbd info testimg_import rbd info testimg_import@snap + IMAGEMETA_AFTER=`rbd image-meta list testimg_import` + [ "$IMAGEMETA_BEFORE" = "$IMAGEMETA_AFTER" ] # compare the contents between testimg and testimg_import rbd export testimg_import ${TMPDIR}/img_import @@ -132,7 +136,7 @@ if rbd help export | grep -q export-format; then rbd import --stripe-count 1000 --stripe-unit 4096 ${TMPDIR}/img testimg rbd export --export-format 2 testimg ${TMPDIR}/img_v2 rbd import --export-format 2 ${TMPDIR}/img_v2 testimg_import - rbd info testimg_import|grep "stripe unit"|awk '{print $3}'|grep 4096 + rbd info testimg_import|grep "stripe unit"|grep -Ei '(4 KiB|4K|4096)' rbd info testimg_import|grep "stripe count"|awk '{print $3}'|grep 1000 rm ${TMPDIR}/img_v2 @@ -158,7 +162,7 @@ dd if=/dev/urandom bs=1M count=1 of=${TMPDIR}/sparse2; truncate ${TMPDIR}/sparse # 1M sparse, 1M data rbd rm sparse1 || true rbd import $RBD_CREATE_ARGS --order 20 ${TMPDIR}/sparse1 -rbd ls -l | grep sparse1 | grep -Ei '(2M|2048k)' +rbd ls -l | grep sparse1 | grep -Ei '(2 MiB|2M|2048k)' [ $tiered -eq 1 -o "$(objects sparse1)" = '1' ] # export, compare contents and on-disk size @@ -170,7 +174,7 @@ rbd rm sparse1 # 1M data, 1M sparse rbd rm sparse2 || true rbd import $RBD_CREATE_ARGS --order 20 ${TMPDIR}/sparse2 -rbd ls -l | grep sparse2 | grep -Ei '(2M|2048k)' +rbd ls -l | grep sparse2 | grep -Ei '(2 MiB|2M|2048k)' [ $tiered -eq 1 -o "$(objects sparse2)" = '0' ] rbd export sparse2 ${TMPDIR}/sparse2.out compare_files_and_ondisk_sizes ${TMPDIR}/sparse2 ${TMPDIR}/sparse2.out @@ -181,7 +185,7 @@ rbd rm sparse2 truncate ${TMPDIR}/sparse1 -s 10M # import from stdin just for fun, verify still sparse rbd import $RBD_CREATE_ARGS --order 20 - sparse1 < ${TMPDIR}/sparse1 -rbd ls -l | grep sparse1 | grep -Ei '(10M|10240k)' +rbd ls -l | grep sparse1 | grep -Ei '(10 MiB|10M|10240k)' [ $tiered -eq 1 -o "$(objects sparse1)" = '1' ] rbd export sparse1 ${TMPDIR}/sparse1.out compare_files_and_ondisk_sizes ${TMPDIR}/sparse1 ${TMPDIR}/sparse1.out @@ -192,7 +196,7 @@ rbd rm sparse1 dd if=/dev/urandom bs=2M count=1 of=${TMPDIR}/sparse2 oflag=append conv=notrunc # again from stding rbd import $RBD_CREATE_ARGS --order 20 - sparse2 < ${TMPDIR}/sparse2 -rbd ls -l | grep sparse2 | grep -Ei '(4M|4096k)' +rbd ls -l | grep sparse2 | grep -Ei '(4 MiB|4M|4096k)' [ $tiered -eq 1 -o "$(objects sparse2)" = '0 2 3' ] rbd export sparse2 ${TMPDIR}/sparse2.out compare_files_and_ondisk_sizes ${TMPDIR}/sparse2 ${TMPDIR}/sparse2.out diff --git a/ceph/qa/workunits/rbd/permissions.sh b/ceph/qa/workunits/rbd/permissions.sh index a435a67bb..fdf91bfc6 100755 --- a/ceph/qa/workunits/rbd/permissions.sh +++ b/ceph/qa/workunits/rbd/permissions.sh @@ -23,11 +23,27 @@ recreate_pools() { delete_users() { (ceph auth del client.volumes || true) >/dev/null 2>&1 (ceph auth del client.images || true) >/dev/null 2>&1 + + (ceph auth del client.snap_none || true) >/dev/null 2>&1 + (ceph auth del client.snap_all || true) >/dev/null 2>&1 + (ceph auth del client.snap_pool || true) >/dev/null 2>&1 + (ceph auth del client.snap_profile_all || true) >/dev/null 2>&1 + (ceph auth del client.snap_profile_pool || true) >/dev/null 2>&1 + + (ceph auth del client.mon_write || true) >/dev/null 2>&1 } create_users() { ceph auth get-or-create client.volumes mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow r class-read pool images, allow rwx pool volumes' >> $KEYRING ceph auth get-or-create client.images mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool images' >> $KEYRING + + ceph auth get-or-create client.snap_none mon 'allow r' >> $KEYRING + ceph auth get-or-create client.snap_all mon 'allow r' osd 'allow w' >> $KEYRING + ceph auth get-or-create client.snap_pool mon 'allow r' osd 'allow w pool=images' >> $KEYRING + ceph auth get-or-create client.snap_profile_all mon 'allow r' osd 'profile rbd' >> $KEYRING + ceph auth get-or-create client.snap_profile_pool mon 'allow r' osd 'profile rbd pool=images' >> $KEYRING + + ceph auth get-or-create client.mon_write mon 'allow *' >> $KEYRING } expect() { @@ -126,9 +142,83 @@ test_volumes_access() { rbd -k $KEYRING --id volumes rm volumes/child } +create_self_managed_snapshot() { + ID=$1 + POOL=$2 + + cat << EOF | CEPH_KEYRING="$KEYRING" python +import rados + +cluster = rados.Rados(conffile="", rados_id="${ID}") +cluster.connect() +ioctx = cluster.open_ioctx("${POOL}") + +snap_id = ioctx.create_self_managed_snap() +print ("Created snap id {}".format(snap_id)) +EOF +} + +remove_self_managed_snapshot() { + ID=$1 + POOL=$2 + + cat << EOF | CEPH_KEYRING="$KEYRING" python +import rados + +cluster1 = rados.Rados(conffile="", rados_id="mon_write") +cluster1.connect() +ioctx1 = cluster1.open_ioctx("${POOL}") + +snap_id = ioctx1.create_self_managed_snap() +print ("Created snap id {}".format(snap_id)) + +cluster2 = rados.Rados(conffile="", rados_id="${ID}") +cluster2.connect() +ioctx2 = cluster2.open_ioctx("${POOL}") + +ioctx2.remove_self_managed_snap(snap_id) +print ("Removed snap id {}".format(snap_id)) +EOF +} + +test_remove_self_managed_snapshots() { + # Ensure users cannot create self-managed snapshots w/o permissions + expect 1 create_self_managed_snapshot snap_none images + expect 1 create_self_managed_snapshot snap_none volumes + + create_self_managed_snapshot snap_all images + create_self_managed_snapshot snap_all volumes + + create_self_managed_snapshot snap_pool images + expect 1 create_self_managed_snapshot snap_pool volumes + + create_self_managed_snapshot snap_profile_all images + create_self_managed_snapshot snap_profile_all volumes + + create_self_managed_snapshot snap_profile_pool images + expect 1 create_self_managed_snapshot snap_profile_pool volumes + + # Ensure users cannot delete self-managed snapshots w/o permissions + expect 1 remove_self_managed_snapshot snap_none images + expect 1 remove_self_managed_snapshot snap_none volumes + + remove_self_managed_snapshot snap_all images + remove_self_managed_snapshot snap_all volumes + + remove_self_managed_snapshot snap_pool images + expect 1 remove_self_managed_snapshot snap_pool volumes + + remove_self_managed_snapshot snap_profile_all images + remove_self_managed_snapshot snap_profile_all volumes + + remove_self_managed_snapshot snap_profile_pool images + expect 1 remove_self_managed_snapshot snap_profile_pool volumes +} + cleanup() { rm -f $KEYRING } + KEYRING=$(mktemp) trap cleanup EXIT ERR HUP INT QUIT @@ -141,6 +231,8 @@ test_images_access recreate_pools test_volumes_access +test_remove_self_managed_snapshots + delete_pools delete_users diff --git a/ceph/qa/workunits/rbd/rbd_mirror.sh b/ceph/qa/workunits/rbd/rbd_mirror.sh index 3e7f844fa..c2308c355 100755 --- a/ceph/qa/workunits/rbd/rbd_mirror.sh +++ b/ceph/qa/workunits/rbd/rbd_mirror.sh @@ -400,7 +400,7 @@ testlog " - rbd_mirroring_resync_after_disconnect config option" set_image_meta ${CLUSTER2} ${POOL} ${image} \ conf_rbd_mirroring_resync_after_disconnect true wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -image_id=$(get_image_id ${CLUSTER1} ${pool} ${image}) +image_id=$(get_image_id ${CLUSTER1} ${POOL} ${image}) disconnect_image ${CLUSTER2} ${POOL} ${image} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' diff --git a/ceph/qa/workunits/rest/test_mgr_rest_api.py b/ceph/qa/workunits/rest/test_mgr_rest_api.py index 7c4335c7b..871f57093 100755 --- a/ceph/qa/workunits/rest/test_mgr_rest_api.py +++ b/ceph/qa/workunits/rest/test_mgr_rest_api.py @@ -72,6 +72,8 @@ screenplay = [ ('get', '/request?page=0', {}), ('delete', '/request', {}), ('get', '/request', {}), + ('patch', '/pool/1', {'pg_num': 128}), + ('patch', '/pool/1', {'pgp_num': 128}), ] for method, endpoint, args in screenplay: diff --git a/ceph/selinux/ceph.te b/ceph/selinux/ceph.te index 0a9349803..a56eb6a55 100644 --- a/ceph/selinux/ceph.te +++ b/ceph/selinux/ceph.te @@ -12,6 +12,7 @@ require { class dir read; class file { getattr read open }; class blk_file { getattr ioctl open read write }; + class capability2 block_suspend; } ######################################## @@ -46,6 +47,7 @@ allow ceph_t self:process { signal_perms }; allow ceph_t self:fifo_file rw_fifo_file_perms; allow ceph_t self:unix_stream_socket create_stream_socket_perms; allow ceph_t self:capability { setuid setgid dac_override }; +allow ceph_t self:capability2 block_suspend; manage_dirs_pattern(ceph_t, ceph_log_t, ceph_log_t) manage_files_pattern(ceph_t, ceph_log_t, ceph_log_t) @@ -103,6 +105,7 @@ fstools_exec(ceph_t) nis_use_ypbind_uncond(ceph_t) storage_raw_rw_fixed_disk(ceph_t) files_manage_generic_locks(ceph_t) +libs_exec_ldconfig(ceph_t) allow ceph_t sysfs_t:dir read; allow ceph_t sysfs_t:file { read getattr open }; diff --git a/ceph/src/.git_version b/ceph/src/.git_version index 823f28dde..fa15181c2 100644 --- a/ceph/src/.git_version +++ b/ceph/src/.git_version @@ -1,2 +1,2 @@ -cad919881333ac92274171586c827e01f554a70a -v12.2.5 +3ec878d1e53e1aeb47a9f619c49d9e7c0aa384d5 +v12.2.7 diff --git a/ceph/src/CMakeLists.txt b/ceph/src/CMakeLists.txt index 7aa8a4392..3d4baae39 100644 --- a/ceph/src/CMakeLists.txt +++ b/ceph/src/CMakeLists.txt @@ -531,6 +531,7 @@ set(libcommon_files common/bit_str.cc osdc/Striper.cc osdc/Objecter.cc + common/compat.cc common/Graylog.cc common/fs_types.cc common/dns_resolve.cc @@ -820,14 +821,15 @@ if (NOT WITH_SYSTEM_ROCKSDB) list(APPEND ROCKSDB_CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}) endif(WITH_CCACHE AND CCACHE_FOUND) + list(APPEND ROCKSDB_CMAKE_ARGS -DWITH_SNAPPY=${SNAPPY_FOUND}) + list(APPEND ROCKSDB_CMAKE_ARGS -DWITH_LZ4=${LZ4_FOUND}) + list(APPEND ROCKSDB_CMAKE_ARGS -DWITH_ZLIB=${ZLIB_FOUND}) + # SSE 4.2 is enabled by default in rocksdb's crc32c. For details refer to # rocksdb/util/crc32c.cc. list(APPEND ROCKSDB_CMAKE_ARGS -DCMAKE_AR=${CMAKE_AR}) list(APPEND ROCKSDB_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}) - - if (CMAKE_CXX_COMPILER_ID STREQUAL Clang) - list(APPEND ROCKSDB_CMAKE_ARGS -DFAIL_ON_WARNINGS=OFF) - endif() + list(APPEND ROCKSDB_CMAKE_ARGS -DFAIL_ON_WARNINGS=OFF) # we use an external project and copy the sources to bin directory to ensure # that object files are built outside of the source tree. diff --git a/ceph/src/arch/arm.c b/ceph/src/arch/arm.c index a8562f596..02f0107b7 100644 --- a/ceph/src/arch/arm.c +++ b/ceph/src/arch/arm.c @@ -4,6 +4,7 @@ /* flags we export */ int ceph_arch_neon = 0; int ceph_arch_aarch64_crc32 = 0; +int ceph_arch_aarch64_pmull = 0; #include @@ -11,48 +12,26 @@ int ceph_arch_aarch64_crc32 = 0; #include #include // ElfW macro +#include #if __arm__ || __aarch64__ #include #endif // __arm__ -static unsigned long get_auxval(unsigned long type) -{ - unsigned long result = 0; - FILE *f = fopen("/proc/self/auxv", "r"); - if (f) { - ElfW(auxv_t) entry; - while (fread(&entry, sizeof(entry), 1, f) == 1) { - if (entry.a_type == type) { - result = entry.a_un.a_val; - break; - } - } - fclose(f); - } - return result; -} - -static unsigned long get_hwcap(void) -{ - return get_auxval(AT_HWCAP); -} - #endif // __linux__ int ceph_arch_arm_probe(void) { -#if __arm__ && __linux__ - ceph_arch_neon = (get_hwcap() & HWCAP_NEON) == HWCAP_NEON; -#elif __aarch64__ && __linux__ - ceph_arch_neon = (get_hwcap() & HWCAP_ASIMD) == HWCAP_ASIMD; -# if defined(HAVE_ARMV8_CRC) && defined(HWCAP_CRC32) - ceph_arch_aarch64_crc32 = (get_hwcap() & HWCAP_CRC32) == HWCAP_CRC32; -# endif -#else - if (0) - get_hwcap(); // make compiler shut up +#if __linux__ + unsigned long hwcap = getauxval(AT_HWCAP); +#if __arm__ + ceph_arch_neon = (hwcap & HWCAP_NEON) == HWCAP_NEON; +#elif __aarch64__ + ceph_arch_neon = (hwcap & HWCAP_ASIMD) == HWCAP_ASIMD; + ceph_arch_aarch64_crc32 = (hwcap & HWCAP_CRC32) == HWCAP_CRC32; + ceph_arch_aarch64_pmull = (hwcap & HWCAP_PMULL) == HWCAP_PMULL; #endif +#endif // __linux__ return 0; } diff --git a/ceph/src/arch/arm.h b/ceph/src/arch/arm.h index 1659b2e94..dacc450b1 100644 --- a/ceph/src/arch/arm.h +++ b/ceph/src/arch/arm.h @@ -7,6 +7,7 @@ extern "C" { extern int ceph_arch_neon; /* true if we have ARM NEON or ASIMD abilities */ extern int ceph_arch_aarch64_crc32; /* true if we have AArch64 CRC32/CRC32C abilities */ +extern int ceph_arch_aarch64_pmull; /* true if we have AArch64 PMULL abilities */ extern int ceph_arch_arm_probe(void); diff --git a/ceph/src/auth/Auth.h b/ceph/src/auth/Auth.h index c5bda0cca..f40296e1a 100644 --- a/ceph/src/auth/Auth.h +++ b/ceph/src/auth/Auth.h @@ -136,6 +136,11 @@ struct AuthAuthorizer { explicit AuthAuthorizer(__u32 p) : protocol(p) {} virtual ~AuthAuthorizer() {} virtual bool verify_reply(bufferlist::iterator& reply) = 0; + virtual bool add_challenge(CephContext *cct, bufferlist& challenge) = 0; +}; + +struct AuthAuthorizerChallenge { + virtual ~AuthAuthorizerChallenge() {} }; diff --git a/ceph/src/auth/AuthAuthorizeHandler.h b/ceph/src/auth/AuthAuthorizeHandler.h index 2e81f14ab..d824ed4e1 100644 --- a/ceph/src/auth/AuthAuthorizeHandler.h +++ b/ceph/src/auth/AuthAuthorizeHandler.h @@ -34,7 +34,9 @@ struct AuthAuthorizeHandler { virtual bool verify_authorizer(CephContext *cct, KeyStore *keys, bufferlist& authorizer_data, bufferlist& authorizer_reply, EntityName& entity_name, uint64_t& global_id, - AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid = NULL) = 0; + AuthCapsInfo& caps_info, CryptoKey& session_key, + uint64_t *auid, + std::unique_ptr *challenge) = 0; virtual int authorizer_session_crypto() = 0; }; diff --git a/ceph/src/auth/cephx/CephxAuthorizeHandler.cc b/ceph/src/auth/cephx/CephxAuthorizeHandler.cc index d455b190a..1a6164a0f 100644 --- a/ceph/src/auth/cephx/CephxAuthorizeHandler.cc +++ b/ceph/src/auth/cephx/CephxAuthorizeHandler.cc @@ -6,9 +6,12 @@ -bool CephxAuthorizeHandler::verify_authorizer(CephContext *cct, KeyStore *keys, - bufferlist& authorizer_data, bufferlist& authorizer_reply, - EntityName& entity_name, uint64_t& global_id, AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid) +bool CephxAuthorizeHandler::verify_authorizer( + CephContext *cct, KeyStore *keys, + bufferlist& authorizer_data, bufferlist& authorizer_reply, + EntityName& entity_name, uint64_t& global_id, AuthCapsInfo& caps_info, + CryptoKey& session_key, uint64_t *auid, + std::unique_ptr *challenge) { bufferlist::iterator iter = authorizer_data.begin(); @@ -19,7 +22,8 @@ bool CephxAuthorizeHandler::verify_authorizer(CephContext *cct, KeyStore *keys, CephXServiceTicketInfo auth_ticket_info; - bool isvalid = cephx_verify_authorizer(cct, keys, iter, auth_ticket_info, authorizer_reply); + bool isvalid = cephx_verify_authorizer(cct, keys, iter, auth_ticket_info, challenge, + authorizer_reply); if (isvalid) { caps_info = auth_ticket_info.ticket.caps; diff --git a/ceph/src/auth/cephx/CephxAuthorizeHandler.h b/ceph/src/auth/cephx/CephxAuthorizeHandler.h index 7246b80c7..8fa40aa71 100644 --- a/ceph/src/auth/cephx/CephxAuthorizeHandler.h +++ b/ceph/src/auth/cephx/CephxAuthorizeHandler.h @@ -23,7 +23,8 @@ struct CephxAuthorizeHandler : public AuthAuthorizeHandler { bool verify_authorizer(CephContext *cct, KeyStore *keys, bufferlist& authorizer_data, bufferlist& authorizer_reply, EntityName& entity_name, uint64_t& global_id, - AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid = NULL) override; + AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid, + std::unique_ptr *challenge) override; int authorizer_session_crypto() override; }; diff --git a/ceph/src/auth/cephx/CephxProtocol.cc b/ceph/src/auth/cephx/CephxProtocol.cc index 5836a33bd..cc5f44964 100644 --- a/ceph/src/auth/cephx/CephxProtocol.cc +++ b/ceph/src/auth/cephx/CephxProtocol.cc @@ -296,7 +296,7 @@ CephXAuthorizer *CephXTicketHandler::build_authorizer(uint64_t global_id) const { CephXAuthorizer *a = new CephXAuthorizer(cct); a->session_key = session_key; - a->nonce = ((uint64_t)rand() << 32) + rand(); + get_random_bytes((char*)&a->nonce, sizeof(a->nonce)); __u8 authorizer_v = 1; ::encode(authorizer_v, a->bl); @@ -304,6 +304,7 @@ CephXAuthorizer *CephXTicketHandler::build_authorizer(uint64_t global_id) const ::encode(service_id, a->bl); ::encode(ticket, a->bl); + a->base_bl = a->bl; CephXAuthorize msg; msg.nonce = a->nonce; @@ -390,7 +391,9 @@ bool cephx_decode_ticket(CephContext *cct, KeyStore *keys, uint32_t service_id, */ bool cephx_verify_authorizer(CephContext *cct, KeyStore *keys, bufferlist::iterator& indata, - CephXServiceTicketInfo& ticket_info, bufferlist& reply_bl) + CephXServiceTicketInfo& ticket_info, + std::unique_ptr *challenge, + bufferlist& reply_bl) { __u8 authorizer_v; uint32_t service_id; @@ -457,6 +460,30 @@ bool cephx_verify_authorizer(CephContext *cct, KeyStore *keys, return false; } + if (challenge) { + auto *c = static_cast(challenge->get()); + if (!auth_msg.have_challenge || !c) { + c = new CephXAuthorizeChallenge; + challenge->reset(c); + get_random_bytes((char*)&c->server_challenge, sizeof(c->server_challenge)); + ldout(cct,10) << __func__ << " adding server_challenge " << c->server_challenge + << dendl; + + encode_encrypt_enc_bl(cct, *c, ticket_info.session_key, reply_bl, error); + if (!error.empty()) { + ldout(cct, 10) << "verify_authorizer: encode_encrypt error: " << error << dendl; + return false; + } + return false; + } + ldout(cct, 10) << __func__ << " got server_challenge+1 " + << auth_msg.server_challenge_plus_one + << " expecting " << c->server_challenge + 1 << dendl; + if (c->server_challenge + 1 != auth_msg.server_challenge_plus_one) { + return false; + } + } + /* * Reply authorizer: * {timestamp + 1}^session_key @@ -493,3 +520,31 @@ bool CephXAuthorizer::verify_reply(bufferlist::iterator& indata) return true; } +bool CephXAuthorizer::add_challenge(CephContext *cct, bufferlist& challenge) +{ + bl = base_bl; + + CephXAuthorize msg; + msg.nonce = nonce; + + auto p = challenge.begin(); + if (!p.end()) { + std::string error; + CephXAuthorizeChallenge ch; + decode_decrypt_enc_bl(cct, ch, session_key, challenge, error); + if (!error.empty()) { + ldout(cct, 0) << "failed to decrypt challenge (" << challenge.length() << " bytes): " + << error << dendl; + return false; + } + msg.have_challenge = true; + msg.server_challenge_plus_one = ch.server_challenge + 1; + } + + std::string error; + if (encode_encrypt(cct, msg, session_key, bl, error)) { + ldout(cct, 0) << __func__ << " failed to encrypt authorizer: " << error << dendl; + return false; + } + return true; +} diff --git a/ceph/src/auth/cephx/CephxProtocol.h b/ceph/src/auth/cephx/CephxProtocol.h index c82206989..b5ec897f3 100644 --- a/ceph/src/auth/cephx/CephxProtocol.h +++ b/ceph/src/auth/cephx/CephxProtocol.h @@ -273,12 +273,14 @@ private: CephContext *cct; public: uint64_t nonce; + bufferlist base_bl; explicit CephXAuthorizer(CephContext *cct_) : AuthAuthorizer(CEPH_AUTH_CEPHX), cct(cct_), nonce(0) {} bool build_authorizer(); bool verify_reply(bufferlist::iterator& reply) override; + bool add_challenge(CephContext *cct, bufferlist& challenge) override; }; @@ -384,17 +386,41 @@ struct CephXServiceTicketInfo { }; WRITE_CLASS_ENCODER(CephXServiceTicketInfo) +struct CephXAuthorizeChallenge : public AuthAuthorizerChallenge { + uint64_t server_challenge; + void encode(bufferlist& bl) const { + __u8 struct_v = 1; + ::encode(struct_v, bl); + ::encode(server_challenge, bl); + } + void decode(bufferlist::iterator& bl) { + __u8 struct_v; + ::decode(struct_v, bl); + ::decode(server_challenge, bl); + } +}; +WRITE_CLASS_ENCODER(CephXAuthorizeChallenge) + struct CephXAuthorize { uint64_t nonce; + bool have_challenge = false; + uint64_t server_challenge_plus_one = 0; void encode(bufferlist& bl) const { - __u8 struct_v = 1; + __u8 struct_v = 2; ::encode(struct_v, bl); ::encode(nonce, bl); + ::encode(have_challenge, bl); + ::encode(server_challenge_plus_one, bl); } void decode(bufferlist::iterator& bl) { __u8 struct_v; ::decode(struct_v, bl); ::decode(nonce, bl); + if (struct_v >= 2) { + ::decode(have_challenge, bl); + ::decode(server_challenge_plus_one, bl); + } + } }; WRITE_CLASS_ENCODER(CephXAuthorize) @@ -409,9 +435,12 @@ bool cephx_decode_ticket(CephContext *cct, KeyStore *keys, /* * Verify authorizer and generate reply authorizer */ -extern bool cephx_verify_authorizer(CephContext *cct, KeyStore *keys, - bufferlist::iterator& indata, - CephXServiceTicketInfo& ticket_info, bufferlist& reply_bl); +extern bool cephx_verify_authorizer( + CephContext *cct, KeyStore *keys, + bufferlist::iterator& indata, + CephXServiceTicketInfo& ticket_info, + std::unique_ptr *challenge, + bufferlist& reply_bl); diff --git a/ceph/src/auth/cephx/CephxServiceHandler.cc b/ceph/src/auth/cephx/CephxServiceHandler.cc index 3184835a1..b06e0080b 100644 --- a/ceph/src/auth/cephx/CephxServiceHandler.cc +++ b/ceph/src/auth/cephx/CephxServiceHandler.cc @@ -152,7 +152,9 @@ int CephxServiceHandler::handle_request(bufferlist::iterator& indata, bufferlist bufferlist tmp_bl; CephXServiceTicketInfo auth_ticket_info; - if (!cephx_verify_authorizer(cct, key_server, indata, auth_ticket_info, tmp_bl)) { + // note: no challenge here. + if (!cephx_verify_authorizer(cct, key_server, indata, auth_ticket_info, nullptr, + tmp_bl)) { ret = -EPERM; break; } diff --git a/ceph/src/auth/cephx/CephxSessionHandler.cc b/ceph/src/auth/cephx/CephxSessionHandler.cc index 5694a22c0..989ba2d35 100644 --- a/ceph/src/auth/cephx/CephxSessionHandler.cc +++ b/ceph/src/auth/cephx/CephxSessionHandler.cc @@ -29,33 +29,75 @@ int CephxSessionHandler::_calc_signature(Message *m, uint64_t *psig) const ceph_msg_header& header = m->get_header(); const ceph_msg_footer& footer = m->get_footer(); - // optimized signature calculation - // - avoid temporary allocated buffers from encode_encrypt[_enc_bl] - // - skip the leading 4 byte wrapper from encode_encrypt - struct { - __u8 v; - __le64 magic; - __le32 len; - __le32 header_crc; - __le32 front_crc; - __le32 middle_crc; - __le32 data_crc; - } __attribute__ ((packed)) sigblock = { - 1, mswab(AUTH_ENC_MAGIC), mswab(4*4), - mswab(header.crc), mswab(footer.front_crc), - mswab(footer.middle_crc), mswab(footer.data_crc) - }; - bufferlist bl_plaintext; - bl_plaintext.append(buffer::create_static(sizeof(sigblock), (char*)&sigblock)); - - bufferlist bl_ciphertext; - if (key.encrypt(cct, bl_plaintext, bl_ciphertext, NULL) < 0) { - lderr(cct) << __func__ << " failed to encrypt signature block" << dendl; - return -1; - } + if (!HAVE_FEATURE(features, CEPHX_V2)) { + // legacy pre-mimic behavior for compatibility + + // optimized signature calculation + // - avoid temporary allocated buffers from encode_encrypt[_enc_bl] + // - skip the leading 4 byte wrapper from encode_encrypt + struct { + __u8 v; + __le64 magic; + __le32 len; + __le32 header_crc; + __le32 front_crc; + __le32 middle_crc; + __le32 data_crc; + } __attribute__ ((packed)) sigblock = { + 1, mswab(AUTH_ENC_MAGIC), mswab(4*4), + mswab(header.crc), mswab(footer.front_crc), + mswab(footer.middle_crc), mswab(footer.data_crc) + }; + + bufferlist bl_plaintext; + bl_plaintext.append(buffer::create_static(sizeof(sigblock), + (char*)&sigblock)); + + bufferlist bl_ciphertext; + if (key.encrypt(cct, bl_plaintext, bl_ciphertext, NULL) < 0) { + lderr(cct) << __func__ << " failed to encrypt signature block" << dendl; + return -1; + } - bufferlist::iterator ci = bl_ciphertext.begin(); - ::decode(*psig, ci); + bufferlist::iterator ci = bl_ciphertext.begin(); + ::decode(*psig, ci); + } else { + // newer mimic+ signatures + struct { + __le32 header_crc; + __le32 front_crc; + __le32 front_len; + __le32 middle_crc; + __le32 middle_len; + __le32 data_crc; + __le32 data_len; + __le32 seq_lower_word; + } __attribute__ ((packed)) sigblock = { + mswab(header.crc), + mswab(footer.front_crc), + mswab(header.front_len), + mswab(footer.middle_crc), + mswab(header.middle_len), + mswab(footer.data_crc), + mswab(header.data_len), + mswab(header.seq) + }; + + bufferlist bl_plaintext; + bl_plaintext.append(buffer::create_static(sizeof(sigblock), + (char*)&sigblock)); + + bufferlist bl_ciphertext; + if (key.encrypt(cct, bl_plaintext, bl_ciphertext, NULL) < 0) { + lderr(cct) << __func__ << " failed to encrypt signature block" << dendl; + return -1; + } + + struct enc { + __le64 a, b, c, d; + } *penc = reinterpret_cast(bl_ciphertext.c_str()); + *psig = penc->a ^ penc->b ^ penc->c ^ penc->d; + } ldout(cct, 10) << __func__ << " seq " << m->get_seq() << " front_crc_ = " << footer.front_crc diff --git a/ceph/src/auth/none/AuthNoneAuthorizeHandler.cc b/ceph/src/auth/none/AuthNoneAuthorizeHandler.cc index 8b55e9e9b..5767eacff 100644 --- a/ceph/src/auth/none/AuthNoneAuthorizeHandler.cc +++ b/ceph/src/auth/none/AuthNoneAuthorizeHandler.cc @@ -17,10 +17,13 @@ #define dout_subsys ceph_subsys_auth -bool AuthNoneAuthorizeHandler::verify_authorizer(CephContext *cct, KeyStore *keys, - bufferlist& authorizer_data, bufferlist& authorizer_reply, - EntityName& entity_name, uint64_t& global_id, AuthCapsInfo& caps_info, CryptoKey& session_key, -uint64_t *auid) +bool AuthNoneAuthorizeHandler::verify_authorizer( + CephContext *cct, KeyStore *keys, + bufferlist& authorizer_data, bufferlist& authorizer_reply, + EntityName& entity_name, uint64_t& global_id, AuthCapsInfo& caps_info, + CryptoKey& session_key, + uint64_t *auid, + std::unique_ptr *challenge) { bufferlist::iterator iter = authorizer_data.begin(); diff --git a/ceph/src/auth/none/AuthNoneAuthorizeHandler.h b/ceph/src/auth/none/AuthNoneAuthorizeHandler.h index b531cfb77..0ce542bf6 100644 --- a/ceph/src/auth/none/AuthNoneAuthorizeHandler.h +++ b/ceph/src/auth/none/AuthNoneAuthorizeHandler.h @@ -23,7 +23,8 @@ struct AuthNoneAuthorizeHandler : public AuthAuthorizeHandler { bool verify_authorizer(CephContext *cct, KeyStore *keys, bufferlist& authorizer_data, bufferlist& authorizer_reply, EntityName& entity_name, uint64_t& global_id, - AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid=NULL) override; + AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid, + std::unique_ptr *challenge) override; int authorizer_session_crypto() override; }; diff --git a/ceph/src/auth/none/AuthNoneProtocol.h b/ceph/src/auth/none/AuthNoneProtocol.h index 8683c5674..662fdb516 100644 --- a/ceph/src/auth/none/AuthNoneProtocol.h +++ b/ceph/src/auth/none/AuthNoneProtocol.h @@ -17,6 +17,8 @@ #include "auth/Auth.h" +class CephContext; + struct AuthNoneAuthorizer : public AuthAuthorizer { AuthNoneAuthorizer() : AuthAuthorizer(CEPH_AUTH_NONE) { } bool build_authorizer(const EntityName &ename, uint64_t global_id) { @@ -27,6 +29,7 @@ struct AuthNoneAuthorizer : public AuthAuthorizer { return 0; } bool verify_reply(bufferlist::iterator& reply) override { return true; } + bool add_challenge(CephContext *cct, bufferlist& ch) override { return true; } }; #endif diff --git a/ceph/src/auth/unknown/AuthUnknownAuthorizeHandler.cc b/ceph/src/auth/unknown/AuthUnknownAuthorizeHandler.cc index 62cb63887..90e00ef57 100644 --- a/ceph/src/auth/unknown/AuthUnknownAuthorizeHandler.cc +++ b/ceph/src/auth/unknown/AuthUnknownAuthorizeHandler.cc @@ -14,10 +14,13 @@ #include "AuthUnknownAuthorizeHandler.h" -bool AuthUnknownAuthorizeHandler::verify_authorizer(CephContext *cct, KeyStore *keys, - bufferlist& authorizer_data, bufferlist& authorizer_reply, - EntityName& entity_name, uint64_t& global_id, AuthCapsInfo& caps_info, CryptoKey& session_key, -uint64_t *auid) +bool AuthUnknownAuthorizeHandler::verify_authorizer( + CephContext *cct, KeyStore *keys, + bufferlist& authorizer_data, bufferlist& authorizer_reply, + EntityName& entity_name, uint64_t& global_id, AuthCapsInfo& caps_info, + CryptoKey& session_key, + uint64_t *auid, + std::unique_ptr *challenge) { // For unknown authorizers, there's nothing to verify. They're "OK" by definition. PLR diff --git a/ceph/src/auth/unknown/AuthUnknownAuthorizeHandler.h b/ceph/src/auth/unknown/AuthUnknownAuthorizeHandler.h index 9795ebfe9..e052af5de 100644 --- a/ceph/src/auth/unknown/AuthUnknownAuthorizeHandler.h +++ b/ceph/src/auth/unknown/AuthUnknownAuthorizeHandler.h @@ -23,7 +23,8 @@ struct AuthUnknownAuthorizeHandler : public AuthAuthorizeHandler { bool verify_authorizer(CephContext *cct, KeyStore *keys, bufferlist& authorizer_data, bufferlist& authorizer_reply, EntityName& entity_name, uint64_t& global_id, - AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid=NULL) override; + AuthCapsInfo& caps_info, CryptoKey& session_key, uint64_t *auid, + std::unique_ptr *challenge) override; int authorizer_session_crypto() override; }; diff --git a/ceph/src/ceph-disk/ceph_disk/main.py b/ceph/src/ceph-disk/ceph_disk/main.py index 63faec6ba..0058f1ac2 100644 --- a/ceph/src/ceph-disk/ceph_disk/main.py +++ b/ceph/src/ceph-disk/ceph_disk/main.py @@ -1528,10 +1528,11 @@ def get_free_partition_index(dev): def check_journal_reqs(args): + log_file = "/var/log/ceph/$cluster-osd-check.log" _, _, allows_journal = command([ 'ceph-osd', '--check-allows-journal', '-i', '0', - '--log-file', '$run_dir/$cluster-osd-check.log', + '--log-file', log_file, '--cluster', args.cluster, '--setuser', get_ceph_user(), '--setgroup', get_ceph_group(), @@ -1539,7 +1540,7 @@ def check_journal_reqs(args): _, _, wants_journal = command([ 'ceph-osd', '--check-wants-journal', '-i', '0', - '--log-file', '$run_dir/$cluster-osd-check.log', + '--log-file', log_file, '--cluster', args.cluster, '--setuser', get_ceph_user(), '--setgroup', get_ceph_group(), @@ -1547,7 +1548,7 @@ def check_journal_reqs(args): _, _, needs_journal = command([ 'ceph-osd', '--check-needs-journal', '-i', '0', - '--log-file', '$run_dir/$cluster-osd-check.log', + '--log-file', log_file, '--cluster', args.cluster, '--setuser', get_ceph_user(), '--setgroup', get_ceph_group(), diff --git a/ceph/src/ceph-volume/ceph_volume/__init__.py b/ceph/src/ceph-volume/ceph_volume/__init__.py index f5500015c..6550db415 100644 --- a/ceph/src/ceph-volume/ceph_volume/__init__.py +++ b/ceph/src/ceph-volume/ceph_volume/__init__.py @@ -1,5 +1,16 @@ from collections import namedtuple + +class UnloadedConfig(object): + """ + This class is used as the default value for conf.ceph so that if + a configuration file is not successfully loaded then it will give + a nice error message when values from the config are used. + """ + def __getattr__(self, *a): + raise RuntimeError("No valid ceph configuration file was loaded.") + conf = namedtuple('config', ['ceph', 'cluster', 'verbosity', 'path', 'log_path']) +conf.ceph = UnloadedConfig() __version__ = "1.0.0" diff --git a/ceph/src/ceph-volume/ceph_volume/api/lvm.py b/ceph/src/ceph-volume/ceph_volume/api/lvm.py index 88f0d2d2a..2f2bd1738 100644 --- a/ceph/src/ceph-volume/ceph_volume/api/lvm.py +++ b/ceph/src/ceph-volume/ceph_volume/api/lvm.py @@ -220,7 +220,7 @@ def get_api_pvs(): /dev/sdv;;07A4F654-4162-4600-8EB3-88D1E42F368D """ - fields = 'pv_name,pv_tags,pv_uuid,vg_name' + fields = 'pv_name,pv_tags,pv_uuid,vg_name,lv_uuid' stdout, stderr, returncode = process.call( ['pvs', '--no-heading', '--readonly', '--separator=";"', '-o', fields] diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py index 37ec6fb72..5e9376e5d 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py @@ -162,10 +162,12 @@ def activate_bluestore(lvs, no_systemd=False): destination = os.path.join(osd_path, 'block.db') process.run(['ln', '-snf', db_device_path, destination]) system.chown(db_device_path) + system.chown(destination) if wal_device_path: destination = os.path.join(osd_path, 'block.wal') process.run(['ln', '-snf', wal_device_path, destination]) system.chown(wal_device_path) + system.chown(destination) if no_systemd is False: # enable the ceph-volume unit for this OSD diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/create.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/create.py index 18c80657e..0f972f9d8 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/create.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/create.py @@ -31,7 +31,7 @@ class Create(object): # activate, which would never need to be rolled back. Activate([]).activate(args) except Exception: - logger.error('lvm activate was unable to complete, while creating the OSD') + logger.exception('lvm activate was unable to complete, while creating the OSD') logger.info('will rollback OSD ID creation') rollback_osd(args, osd_id) raise diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/listing.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/listing.py index d1d96d7ad..a84a39c18 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/listing.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/listing.py @@ -49,6 +49,9 @@ def pretty_report(report): value=value ) ) + output.append( + device_metadata_item_template.format(tag_name='devices', value=','.join(device['devices']))) + print(''.join(output)) @@ -74,6 +77,30 @@ class List(object): def __init__(self, argv): self.argv = argv + @property + def pvs(self): + """ + To avoid having to make an LVM API call for every single item being + reported, the call gets set only once, using that stored call for + subsequent calls + """ + if getattr(self, '_pvs', None) is not None: + return self._pvs + self._pvs = api.get_api_pvs() + return self._pvs + + def match_devices(self, lv_uuid): + """ + It is possible to have more than one PV reported *with the same name*, + to avoid incorrect or duplicate contents we correlated the lv uuid to + the one on the physical device. + """ + devices = [] + for device in self.pvs: + if device.get('lv_uuid') == lv_uuid: + devices.append(device['pv_name']) + return devices + @decorators.needs_root def list(self, args): # ensure everything is up to date before calling out @@ -152,6 +179,7 @@ class List(object): return self.full_report(lvs=lvs) if lv: + try: _id = lv.tags['ceph.osd_id'] except KeyError: @@ -159,9 +187,9 @@ class List(object): return report report.setdefault(_id, []) - report[_id].append( - lv.as_dict() - ) + lv_report = lv.as_dict() + lv_report['devices'] = self.match_devices(lv.lv_uuid) + report[_id].append(lv_report) else: # this has to be a journal/wal/db device (not a logical volume) so try @@ -202,9 +230,9 @@ class List(object): continue report.setdefault(_id, []) - report[_id].append( - lv.as_dict() - ) + lv_report = lv.as_dict() + lv_report['devices'] = self.match_devices(lv.lv_uuid) + report[_id].append(lv_report) for device_type in ['journal', 'block', 'wal', 'db']: device_uuid = lv.tags.get('ceph.%s_uuid' % device_type) diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/prepare.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/prepare.py index 17ac5e1b3..2369cb6f3 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/prepare.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/prepare.py @@ -215,7 +215,7 @@ class Prepare(object): try: self.prepare(args) except Exception: - logger.error('lvm prepare was unable to complete') + logger.exception('lvm prepare was unable to complete') logger.info('will rollback OSD ID creation') rollback_osd(args, self.osd_id) raise diff --git a/ceph/src/ceph-volume/ceph_volume/tests/conftest.py b/ceph/src/ceph-volume/ceph_volume/tests/conftest.py index ba3395bfb..b1f858138 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/conftest.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/conftest.py @@ -10,9 +10,15 @@ class Capture(object): self.a = a self.kw = kw self.calls = [] + self.return_values = kw.get('return_values', False) + self.always_returns = kw.get('always_returns', False) def __call__(self, *a, **kw): self.calls.append({'args': a, 'kwargs': kw}) + if self.always_returns: + return self.always_returns + if self.return_values: + return self.return_values.pop() class Factory(object): @@ -41,7 +47,7 @@ def fake_run(monkeypatch): @pytest.fixture def fake_call(monkeypatch): - fake_call = Capture() + fake_call = Capture(always_returns=([], [], 0)) monkeypatch.setattr('ceph_volume.process.call', fake_call) return fake_call @@ -51,10 +57,12 @@ def stub_call(monkeypatch): """ Monkeypatches process.call, so that a caller can add behavior to the response """ - def apply(return_value): - monkeypatch.setattr( - 'ceph_volume.process.call', - lambda *a, **kw: return_value) + def apply(return_values): + if isinstance(return_values, tuple): + return_values = [return_values] + stubbed_call = Capture(return_values=return_values) + monkeypatch.setattr('ceph_volume.process.call', stubbed_call) + return stubbed_call return apply diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_listing.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_listing.py index b780ea2e9..a49a3e9e6 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_listing.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_listing.py @@ -22,12 +22,16 @@ class TestPrettyReport(object): assert stdout == '\n' def test_type_and_path_are_reported(self, capsys): - lvm.listing.pretty_report({0: [{'type': 'data', 'path': '/dev/sda1'}]}) + lvm.listing.pretty_report({0: [ + {'type': 'data', 'path': '/dev/sda1', 'devices': ['/dev/sda']} + ]}) stdout, stderr = capsys.readouterr() assert '[data] /dev/sda1' in stdout def test_osd_id_header_is_reported(self, capsys): - lvm.listing.pretty_report({0: [{'type': 'data', 'path': '/dev/sda1'}]}) + lvm.listing.pretty_report({0: [ + {'type': 'data', 'path': '/dev/sda1', 'devices': ['/dev/sda']} + ]}) stdout, stderr = capsys.readouterr() assert '====== osd.0 =======' in stdout @@ -36,12 +40,20 @@ class TestPrettyReport(object): {0: [{ 'type': 'data', 'path': '/dev/sda1', - 'tags': {'ceph.osd_id': '0'} + 'tags': {'ceph.osd_id': '0'}, + 'devices': ['/dev/sda'], }]} ) stdout, stderr = capsys.readouterr() assert 'osd id' in stdout + def test_devices_are_comma_separated(self, capsys): + lvm.listing.pretty_report({0: [ + {'type': 'data', 'path': '/dev/sda1', 'devices': ['/dev/sda', '/dev/sdb1']} + ]}) + stdout, stderr = capsys.readouterr() + assert '/dev/sda,/dev/sdb1' in stdout + class TestList(object): @@ -155,22 +167,85 @@ class TestSingleReport(object): # ceph lvs are detected by looking into its tags tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data' lv = api.Volume( - lv_name='lv', vg_name='VolGroup', lv_path='/dev/VolGroup/lv', lv_tags=tags) + lv_name='lv', vg_name='VolGroup', + lv_uuid='aaaa', lv_path='/dev/VolGroup/lv', lv_tags=tags + ) volumes.append(lv) monkeypatch.setattr(lvm.listing.api, 'Volumes', lambda: volumes) result = lvm.listing.List([]).single_report('VolGroup/lv') assert result['0'][0]['name'] == 'lv' assert result['0'][0]['lv_tags'] == tags assert result['0'][0]['path'] == '/dev/VolGroup/lv' + assert result['0'][0]['devices'] == [] def test_report_a_ceph_journal_device(self, volumes, monkeypatch): # ceph lvs are detected by looking into its tags tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data,ceph.journal_device=/dev/sda1' lv = api.Volume( - lv_name='lv', vg_name='VolGroup', lv_path='/dev/VolGroup/lv', lv_tags=tags) + lv_name='lv', vg_name='VolGroup', lv_path='/dev/VolGroup/lv', + lv_uuid='aaa', lv_tags=tags) volumes.append(lv) monkeypatch.setattr(lvm.listing.api, 'Volumes', lambda: volumes) result = lvm.listing.List([]).single_report('/dev/sda1') assert result['0'][0]['tags'] == {'PARTUUID': 'x'} assert result['0'][0]['type'] == 'journal' assert result['0'][0]['path'] == '/dev/sda1' + + def test_report_a_ceph_lv_with_devices(self, volumes, monkeypatch): + tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data' + lv = api.Volume( + lv_name='lv', vg_name='VolGroup', + lv_uuid='aaaa', lv_path='/dev/VolGroup/lv', lv_tags=tags + ) + volumes.append(lv) + monkeypatch.setattr(lvm.listing.api, 'Volumes', lambda: volumes) + listing = lvm.listing.List([]) + listing._pvs = [ + {'lv_uuid': 'aaaa', 'pv_name': '/dev/sda1', 'pv_tags': '', 'pv_uuid': ''}, + {'lv_uuid': 'aaaa', 'pv_name': '/dev/sdb1', 'pv_tags': '', 'pv_uuid': ''}, + ] + result = listing.single_report('VolGroup/lv') + assert result['0'][0]['name'] == 'lv' + assert result['0'][0]['lv_tags'] == tags + assert result['0'][0]['path'] == '/dev/VolGroup/lv' + assert result['0'][0]['devices'] == ['/dev/sda1', '/dev/sdb1'] + + def test_report_a_ceph_lv_with_no_matching_devices(self, volumes, monkeypatch): + tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data' + lv = api.Volume( + lv_name='lv', vg_name='VolGroup', + lv_uuid='aaaa', lv_path='/dev/VolGroup/lv', lv_tags=tags + ) + volumes.append(lv) + monkeypatch.setattr(lvm.listing.api, 'Volumes', lambda: volumes) + listing = lvm.listing.List([]) + listing._pvs = [ + {'lv_uuid': 'ffff', 'pv_name': '/dev/sda1', 'pv_tags': '', 'pv_uuid': ''}, + {'lv_uuid': 'ffff', 'pv_name': '/dev/sdb1', 'pv_tags': '', 'pv_uuid': ''}, + ] + result = listing.single_report('VolGroup/lv') + assert result['0'][0]['name'] == 'lv' + assert result['0'][0]['lv_tags'] == tags + assert result['0'][0]['path'] == '/dev/VolGroup/lv' + assert result['0'][0]['devices'] == [] + + +class TestListingPVs(object): + + def setup(self): + self.default_pvs = [ + {'lv_uuid': 'ffff', 'pv_name': '/dev/sda1', 'pv_tags': '', 'pv_uuid': ''}, + {'lv_uuid': 'ffff', 'pv_name': '/dev/sdb1', 'pv_tags': '', 'pv_uuid': ''}, + ] + + def test_pvs_is_unset(self, monkeypatch): + monkeypatch.setattr(lvm.listing.api, 'get_api_pvs', lambda: self.default_pvs) + listing = lvm.listing.List([]) + assert listing.pvs == self.default_pvs + + def test_pvs_is_set(self, monkeypatch): + # keep it patched so that we can fail if this gets returned + monkeypatch.setattr(lvm.listing.api, 'get_api_pvs', lambda: self.default_pvs) + listing = lvm.listing.List([]) + listing._pvs = [] + assert listing.pvs == [] diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/group_vars/all index 2050c5990..37e87be31 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/group_vars/all @@ -10,7 +10,7 @@ osd_objectstore: "bluestore" osd_scenario: lvm ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false lvm_volumes: - data: data-lv1 data_vg: test_group diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/dmcrypt/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/dmcrypt/group_vars/all index 90ad04d42..6f36f4922 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/dmcrypt/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/dmcrypt/group_vars/all @@ -11,7 +11,7 @@ osd_objectstore: "bluestore" osd_scenario: lvm ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false lvm_volumes: - data: data-lv1 data_vg: test_group diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/dmcrypt/test.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/dmcrypt/test.yml index 70c018aa7..bbdc1035d 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/dmcrypt/test.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/dmcrypt/test.yml @@ -1,4 +1,3 @@ - - hosts: osds become: yes tasks: @@ -8,9 +7,26 @@ name: ceph-osd@2 state: stopped + - name: stop ceph-osd@0 daemon + service: + name: ceph-osd@0 + state: stopped + +- hosts: mons + become: yes + tasks: + - name: destroy osd.2 command: "ceph osd destroy osd.2 --yes-i-really-mean-it" + - name: destroy osd.0 + command: "ceph osd destroy osd.0 --yes-i-really-mean-it" + +- hosts: osds + become: yes + tasks: + + # osd.2 device - name: zap /dev/sdd1 command: "ceph-volume lvm zap /dev/sdd1 --destroy" environment: @@ -21,14 +37,7 @@ environment: CEPH_VOLUME_DEBUG: 1 - - name: stop ceph-osd@0 daemon - service: - name: ceph-osd@0 - state: stopped - - - name: destroy osd.0 - command: "ceph osd destroy osd.0 --yes-i-really-mean-it" - + # osd.0 lv - name: zap test_group/data-lv1 command: "ceph-volume lvm zap test_group/data-lv1" environment: @@ -44,9 +53,20 @@ name: ceph-osd@0 state: stopped + +- hosts: mons + become: yes + tasks: + - name: destroy osd.0 command: "ceph osd destroy osd.0 --yes-i-really-mean-it" + +- hosts: osds + become: yes + tasks: + + - name: zap test_group/data-lv1 command: "ceph-volume lvm zap test_group/data-lv1" environment: diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/group_vars/all index f90d233d8..af647e911 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/group_vars/all @@ -10,7 +10,7 @@ osd_objectstore: "filestore" osd_scenario: lvm ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false # test-volume is created by tests/functional/lvm_setup.yml from /dev/sda lvm_volumes: - data: data-lv1 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/dmcrypt/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/dmcrypt/group_vars/all index 871523a23..8cd6c48c1 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/dmcrypt/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/dmcrypt/group_vars/all @@ -11,7 +11,7 @@ osd_objectstore: "filestore" osd_scenario: lvm ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false # test-volume is created by tests/functional/lvm_setup.yml from /dev/sda lvm_volumes: - data: data-lv1 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/dmcrypt/test.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/dmcrypt/test.yml index fc3e38c7a..49f37d20a 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/dmcrypt/test.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/dmcrypt/test.yml @@ -8,9 +8,28 @@ name: ceph-osd@2 state: stopped + - name: stop ceph-osd@0 daemon + service: + name: ceph-osd@0 + state: stopped + + +- hosts: mons + become: yes + tasks: + - name: destroy osd.2 command: "ceph osd destroy osd.2 --yes-i-really-mean-it" + - name: destroy osd.0 + command: "ceph osd destroy osd.0 --yes-i-really-mean-it" + + +- hosts: osds + become: yes + tasks: + + # osd.2 device - name: zap /dev/sdd1 command: "ceph-volume lvm zap /dev/sdd1 --destroy" environment: @@ -26,14 +45,7 @@ environment: CEPH_VOLUME_DEBUG: 1 - - name: stop ceph-osd@0 daemon - service: - name: ceph-osd@0 - state: stopped - - - name: destroy osd.0 - command: "ceph osd destroy osd.0 --yes-i-really-mean-it" - + # osd.0 lv - name: zap test_group/data-lv1 command: "ceph-volume lvm zap test_group/data-lv1" environment: diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml index ec1e11b83..eb0ef32cd 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml @@ -8,9 +8,28 @@ name: ceph-osd@2 state: stopped + - name: stop ceph-osd@0 daemon + service: + name: ceph-osd@0 + state: stopped + + +- hosts: mons + become: yes + tasks: + - name: destroy osd.2 command: "ceph osd destroy osd.2 --yes-i-really-mean-it" + - name: destroy osd.0 + command: "ceph osd destroy osd.0 --yes-i-really-mean-it" + + +- hosts: osds + become: yes + tasks: + + # osd.2 device - name: zap /dev/sdd1 command: "ceph-volume lvm zap /dev/sdd1 --destroy" environment: @@ -21,14 +40,7 @@ environment: CEPH_VOLUME_DEBUG: 1 - - name: stop ceph-osd@0 daemon - service: - name: ceph-osd@0 - state: stopped - - - name: destroy osd.0 - command: "ceph osd destroy osd.0 --yes-i-really-mean-it" - + # osd.0 device - name: zap test_group/data-lv1 command: "ceph-volume lvm zap test_group/data-lv1" environment: diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml index fc3e38c7a..d37efe19d 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml @@ -8,14 +8,34 @@ name: ceph-osd@2 state: stopped + - name: stop ceph-osd@0 daemon + service: + name: ceph-osd@0 + state: stopped + + +- hosts: mons + become: yes + tasks: + - name: destroy osd.2 command: "ceph osd destroy osd.2 --yes-i-really-mean-it" + - name: destroy osd.0 + command: "ceph osd destroy osd.0 --yes-i-really-mean-it" + + +- hosts: osds + become: yes + tasks: + + # osd.2 device - name: zap /dev/sdd1 command: "ceph-volume lvm zap /dev/sdd1 --destroy" environment: CEPH_VOLUME_DEBUG: 1 + # osd.2 journal - name: zap /dev/sdd2 command: "ceph-volume lvm zap /dev/sdd2 --destroy" environment: @@ -26,19 +46,13 @@ environment: CEPH_VOLUME_DEBUG: 1 - - name: stop ceph-osd@0 daemon - service: - name: ceph-osd@0 - state: stopped - - - name: destroy osd.0 - command: "ceph osd destroy osd.0 --yes-i-really-mean-it" - + # osd.0 data lv - name: zap test_group/data-lv1 command: "ceph-volume lvm zap test_group/data-lv1" environment: CEPH_VOLUME_DEBUG: 1 + # osd.0 journal device - name: zap /dev/sdc1 command: "ceph-volume lvm zap /dev/sdc1 --destroy" environment: diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini index 429b467f4..5a2ff02a1 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini @@ -20,6 +20,7 @@ deps= ansible==2.4.1 testinfra==1.7.1 pytest-xdist + notario>=0.0.13 changedir= # plain/unencrypted centos7-filestore-create: {toxinidir}/centos7/filestore/create @@ -39,6 +40,9 @@ changedir= centos7-bluestore-prepare_activate: {toxinidir}/xenial/bluestore/prepare_activate commands= git clone -b {env:CEPH_ANSIBLE_BRANCH:master} --single-branch https://github.com/ceph/ceph-ansible.git {envdir}/tmp/ceph-ansible + # XXX Ideally we should be able to consume the requirements for ceph-ansible directly, + # but the master branch doesn't pin dependencies so we can't guarantee to work correctly + #pip install -r {envdir}/tmp/ceph-ansible/requirements.txt vagrant up --no-provision {posargs:--provider=virtualbox} bash {toxinidir}/../scripts/generate_ssh_config.sh {changedir} diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/group_vars/all index 2050c5990..37e87be31 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/group_vars/all @@ -10,7 +10,7 @@ osd_objectstore: "bluestore" osd_scenario: lvm ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false lvm_volumes: - data: data-lv1 data_vg: test_group diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/dmcrypt/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/dmcrypt/group_vars/all index 90ad04d42..6f36f4922 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/dmcrypt/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/dmcrypt/group_vars/all @@ -11,7 +11,7 @@ osd_objectstore: "bluestore" osd_scenario: lvm ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false lvm_volumes: - data: data-lv1 data_vg: test_group diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/dmcrypt/test.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/dmcrypt/test.yml index 9a60ab2e0..f93ff2304 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/dmcrypt/test.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/dmcrypt/test.yml @@ -1,4 +1,3 @@ - - hosts: osds become: yes tasks: @@ -8,33 +7,44 @@ name: ceph-osd@2 state: stopped - - name: destroy osd.2 + - name: stop ceph-osd@0 daemon + service: + name: ceph-osd@0 + state: stopped + +- hosts: mons + become: yes + tasks: + + - name: destroy osd.2 command: "ceph osd destroy osd.2 --yes-i-really-mean-it" - - name: zap /dev/sdd1 + - name: destroy osd.0 + command: "ceph osd destroy osd.0 --yes-i-really-mean-it" + + +- hosts: osds + become: yes + tasks: + + # osd.2 device + - name: zap /dev/sdd1 command: "ceph-volume lvm zap /dev/sdd1 --destroy" environment: CEPH_VOLUME_DEBUG: 1 - - name: redeploy osd.2 using /dev/sdd1 + - name: redeploy osd.2 using /dev/sdd1 command: "ceph-volume lvm create --bluestore --data /dev/sdd1 --osd-id 2" environment: CEPH_VOLUME_DEBUG: 1 - - name: stop ceph-osd@0 daemon - service: - name: ceph-osd@0 - state: stopped - - - name: destroy osd.0 - command: "ceph osd destroy osd.0 --yes-i-really-mean-it" - - - name: zap test_group/data-lv1 + # osd.0 lv + - name: zap test_group/data-lv1 command: "ceph-volume lvm zap test_group/data-lv1" environment: CEPH_VOLUME_DEBUG: 1 - - name: redeploy osd.0 using test_group/data-lv1 + - name: redeploy osd.0 using test_group/data-lv1 command: "ceph-volume lvm create --bluestore --data test_group/data-lv1 --osd-id 0" environment: CEPH_VOLUME_DEBUG: 1 @@ -44,9 +54,19 @@ name: ceph-osd@0 state: stopped + +- hosts: mons + become: yes + tasks: + - name: destroy osd.0 command: "ceph osd destroy osd.0 --yes-i-really-mean-it" + +- hosts: osds + become: yes + tasks: + - name: zap test_group/data-lv1 command: "ceph-volume lvm zap test_group/data-lv1" environment: diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/group_vars/all index f90d233d8..af647e911 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/group_vars/all @@ -10,7 +10,7 @@ osd_objectstore: "filestore" osd_scenario: lvm ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false # test-volume is created by tests/functional/lvm_setup.yml from /dev/sda lvm_volumes: - data: data-lv1 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/dmcrypt/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/dmcrypt/group_vars/all index 871523a23..8cd6c48c1 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/dmcrypt/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/dmcrypt/group_vars/all @@ -11,7 +11,7 @@ osd_objectstore: "filestore" osd_scenario: lvm ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false # test-volume is created by tests/functional/lvm_setup.yml from /dev/sda lvm_volumes: - data: data-lv1 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/dmcrypt/test.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/dmcrypt/test.yml index fc3e38c7a..49f37d20a 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/dmcrypt/test.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/dmcrypt/test.yml @@ -8,9 +8,28 @@ name: ceph-osd@2 state: stopped + - name: stop ceph-osd@0 daemon + service: + name: ceph-osd@0 + state: stopped + + +- hosts: mons + become: yes + tasks: + - name: destroy osd.2 command: "ceph osd destroy osd.2 --yes-i-really-mean-it" + - name: destroy osd.0 + command: "ceph osd destroy osd.0 --yes-i-really-mean-it" + + +- hosts: osds + become: yes + tasks: + + # osd.2 device - name: zap /dev/sdd1 command: "ceph-volume lvm zap /dev/sdd1 --destroy" environment: @@ -26,14 +45,7 @@ environment: CEPH_VOLUME_DEBUG: 1 - - name: stop ceph-osd@0 daemon - service: - name: ceph-osd@0 - state: stopped - - - name: destroy osd.0 - command: "ceph osd destroy osd.0 --yes-i-really-mean-it" - + # osd.0 lv - name: zap test_group/data-lv1 command: "ceph-volume lvm zap test_group/data-lv1" environment: diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/group_vars/all index 560c8b03b..c265e783b 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/group_vars/all @@ -9,7 +9,7 @@ journal_size: 100 osd_objectstore: "bluestore" ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false os_tuning_params: - { name: kernel.pid_max, value: 4194303 } - { name: fs.file-max, value: 26234859 } diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/group_vars/all index 83d24e0a0..885c2c82f 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/group_vars/all @@ -10,7 +10,7 @@ journal_size: 100 osd_objectstore: "bluestore" ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false os_tuning_params: - { name: kernel.pid_max, value: 4194303 } - { name: fs.file-max, value: 26234859 } diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/group_vars/all index 674abc939..30bcf5be7 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/group_vars/all @@ -10,7 +10,7 @@ journal_size: 100 osd_objectstore: "bluestore" ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false os_tuning_params: - { name: kernel.pid_max, value: 4194303 } - { name: fs.file-max, value: 26234859 } diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/group_vars/all index 8902bdda3..7ab573b07 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/group_vars/all @@ -9,7 +9,7 @@ journal_size: 100 osd_objectstore: "filestore" ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false os_tuning_params: - { name: kernel.pid_max, value: 4194303 } - { name: fs.file-max, value: 26234859 } diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/dmcrypt-luks/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/dmcrypt-luks/group_vars/all index c8ba0ba4c..a27cfbad0 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/dmcrypt-luks/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/dmcrypt-luks/group_vars/all @@ -10,7 +10,7 @@ journal_size: 100 osd_objectstore: "filestore" ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false os_tuning_params: - { name: kernel.pid_max, value: 4194303 } - { name: fs.file-max, value: 26234859 } diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/dmcrypt-plain/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/dmcrypt-plain/group_vars/all index 29e82024c..edac61b20 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/dmcrypt-plain/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/dmcrypt-plain/group_vars/all @@ -10,7 +10,7 @@ journal_size: 100 osd_objectstore: "filestore" ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false os_tuning_params: - { name: kernel.pid_max, value: 4194303 } - { name: fs.file-max, value: 26234859 } diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini index 55d8c81d0..d5e9c33ea 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini @@ -21,6 +21,7 @@ deps= ansible==2.4.1 testinfra==1.7.1 pytest-xdist + notario>=0.0.13 changedir= centos7-filestore-activate: {toxinidir}/centos7/filestore/activate centos7-bluestore-activate: {toxinidir}/centos7/bluestore/activate @@ -36,6 +37,9 @@ changedir= centos7-filestore-dmcrypt_luks: {toxinidir}/centos7/filestore/dmcrypt-luks commands= git clone -b {env:CEPH_ANSIBLE_BRANCH:master} --single-branch https://github.com/ceph/ceph-ansible.git {envdir}/tmp/ceph-ansible + # XXX Ideally we should be able to consume the requirements for ceph-ansible directly, + # but the master branch doesn't pin dependencies so we can't guarantee to work correctly + #pip install -r {envdir}/tmp/ceph-ansible/requirements.txt vagrant up --no-provision {posargs:--provider=virtualbox} bash {toxinidir}/../scripts/generate_ssh_config.sh {changedir} diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/group_vars/all index 560c8b03b..c265e783b 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/group_vars/all @@ -9,7 +9,7 @@ journal_size: 100 osd_objectstore: "bluestore" ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false os_tuning_params: - { name: kernel.pid_max, value: 4194303 } - { name: fs.file-max, value: 26234859 } diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/dmcrypt-luks/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/dmcrypt-luks/group_vars/all index 83d24e0a0..885c2c82f 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/dmcrypt-luks/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/dmcrypt-luks/group_vars/all @@ -10,7 +10,7 @@ journal_size: 100 osd_objectstore: "bluestore" ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false os_tuning_params: - { name: kernel.pid_max, value: 4194303 } - { name: fs.file-max, value: 26234859 } diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/dmcrypt-plain/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/dmcrypt-plain/group_vars/all index 674abc939..30bcf5be7 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/dmcrypt-plain/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/dmcrypt-plain/group_vars/all @@ -10,7 +10,7 @@ journal_size: 100 osd_objectstore: "bluestore" ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false os_tuning_params: - { name: kernel.pid_max, value: 4194303 } - { name: fs.file-max, value: 26234859 } diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/group_vars/all index 8902bdda3..7ab573b07 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/group_vars/all @@ -9,7 +9,7 @@ journal_size: 100 osd_objectstore: "filestore" ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false os_tuning_params: - { name: kernel.pid_max, value: 4194303 } - { name: fs.file-max, value: 26234859 } diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/dmcrypt-luks/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/dmcrypt-luks/group_vars/all index c8ba0ba4c..a27cfbad0 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/dmcrypt-luks/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/dmcrypt-luks/group_vars/all @@ -10,7 +10,7 @@ journal_size: 100 osd_objectstore: "filestore" ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false os_tuning_params: - { name: kernel.pid_max, value: 4194303 } - { name: fs.file-max, value: 26234859 } diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/dmcrypt-plain/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/dmcrypt-plain/group_vars/all index 29e82024c..edac61b20 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/dmcrypt-plain/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/dmcrypt-plain/group_vars/all @@ -10,7 +10,7 @@ journal_size: 100 osd_objectstore: "filestore" ceph_origin: 'repository' ceph_repository: 'dev' -copy_admin_key: true +copy_admin_key: false os_tuning_params: - { name: kernel.pid_max, value: 4194303 } - { name: fs.file-max, value: 26234859 } diff --git a/ceph/src/ceph-volume/ceph_volume/tests/util/test_prepare.py b/ceph/src/ceph-volume/ceph_volume/tests/util/test_prepare.py index bf825ff46..e702e052d 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/util/test_prepare.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/util/test_prepare.py @@ -262,3 +262,31 @@ class TestNormalizeFlags(object): def test_normalize_strings_duplicate_flags(self, flags): result = sorted(prepare._normalize_mount_flags(flags, extras=['discard','rw']).split(',')) assert ','.join(result) == 'auto,discard,exec,rw' + + +class TestMkfsBluestore(object): + + def test_non_zero_exit_status(self, stub_call, monkeypatch): + conf.cluster = 'ceph' + monkeypatch.setattr('ceph_volume.util.prepare.system.chown', lambda x: True) + stub_call(([], [], 1)) + with pytest.raises(RuntimeError) as error: + prepare.osd_mkfs_bluestore('1', 'asdf-1234', keyring='keyring') + assert "Command failed with exit code 1" in str(error) + + def test_non_zero_exit_formats_command_correctly(self, stub_call, monkeypatch): + conf.cluster = 'ceph' + monkeypatch.setattr('ceph_volume.util.prepare.system.chown', lambda x: True) + stub_call(([], [], 1)) + with pytest.raises(RuntimeError) as error: + prepare.osd_mkfs_bluestore('1', 'asdf-1234', keyring='keyring') + expected = ' '.join([ + 'ceph-osd', + '--cluster', + 'ceph', + '--osd-objectstore', 'bluestore', '--mkfs', + '-i', '1', '--monmap', '/var/lib/ceph/osd/ceph-1/activate.monmap', + '--keyfile', '-', '--osd-data', '/var/lib/ceph/osd/ceph-1/', + '--osd-uuid', 'asdf-1234', + '--setuser', 'ceph', '--setgroup', 'ceph']) + assert expected in str(error) diff --git a/ceph/src/ceph-volume/ceph_volume/util/prepare.py b/ceph/src/ceph-volume/ceph_volume/util/prepare.py index d02c570fe..d1cddf073 100644 --- a/ceph/src/ceph-volume/ceph_volume/util/prepare.py +++ b/ceph/src/ceph-volume/ceph_volume/util/prepare.py @@ -322,7 +322,9 @@ def osd_mkfs_bluestore(osd_id, fsid, keyring=None, wal=False, db=False): command = base_command + supplementary_command - process.call(command, stdin=keyring, show_command=True) + _, _, returncode = process.call(command, stdin=keyring, show_command=True) + if returncode != 0: + raise RuntimeError('Command failed with exit code %s: %s' % (returncode, ' '.join(command))) def osd_mkfs_filestore(osd_id, fsid): diff --git a/ceph/src/ceph-volume/ceph_volume/util/system.py b/ceph/src/ceph-volume/ceph_volume/util/system.py index 0ba46d362..b4b7d17c4 100644 --- a/ceph/src/ceph-volume/ceph_volume/util/system.py +++ b/ceph/src/ceph-volume/ceph_volume/util/system.py @@ -87,6 +87,7 @@ def chown(path, recursive=True): """ uid, gid = get_ceph_user_ids() if os.path.islink(path): + process.run(['chown', '-h', 'ceph:ceph', path]) path = os.path.realpath(path) if recursive: process.run(['chown', '-R', 'ceph:ceph', path]) diff --git a/ceph/src/ceph_fuse.cc b/ceph/src/ceph_fuse.cc index 851425af0..77f4702d1 100644 --- a/ceph/src/ceph_fuse.cc +++ b/ceph/src/ceph_fuse.cc @@ -93,6 +93,19 @@ int main(int argc, const char **argv, const char *envp[]) { filer_flags |= CEPH_OSD_FLAG_LOCALIZE_READS; } else if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) { usage(); + } else if (ceph_argparse_flag(args, i, "-V", (char*)nullptr)) { + const char* tmpargv[] = { + "ceph-fuse", + "-V" + }; + + struct fuse_args fargs = FUSE_ARGS_INIT(2, (char**)tmpargv); + if (fuse_parse_cmdline(&fargs, nullptr, nullptr, nullptr) == -1) { + derr << "fuse_parse_cmdline failed." << dendl; + } + assert(fargs.allocated); + fuse_opt_free_args(&fargs); + exit(0); } else { ++i; } @@ -254,9 +267,11 @@ int main(int argc, const char **argv, const char *envp[]) { r = client->mount(g_conf->client_mountpoint.c_str(), perms, g_ceph_context->_conf->fuse_require_active_mds); if (r < 0) { - if (r == CEPH_FUSE_NO_MDS_UP) + if (r == CEPH_FUSE_NO_MDS_UP) { cerr << "ceph-fuse[" << getpid() << "]: probably no MDS server is up?" << std::endl; - cerr << "ceph-fuse[" << getpid() << "]: ceph mount failed with " << cpp_strerror(-r) << std::endl; + } + cerr << "ceph-fuse[" << getpid() << "]: ceph mount failed with " << cpp_strerror(-r) << std::endl;r = EXIT_FAILURE; + r = EXIT_FAILURE; goto out_shutdown; } diff --git a/ceph/src/client/Client.cc b/ceph/src/client/Client.cc index 1fd435839..41eff1b4b 100644 --- a/ceph/src/client/Client.cc +++ b/ceph/src/client/Client.cc @@ -236,7 +236,6 @@ Client::Client(Messenger *m, MonClient *mc, Objecter *objecter_) remount_cb(NULL), ino_invalidate_cb(NULL), dentry_invalidate_cb(NULL), - getgroups_cb(NULL), umask_cb(NULL), can_invalidate_dentries(false), async_ino_invalidator(m->cct), @@ -902,8 +901,10 @@ Inode * Client::add_update_inode(InodeStat *st, utime_t from, add_update_cap(in, session, st->cap.cap_id, st->cap.caps, st->cap.seq, st->cap.mseq, inodeno_t(st->cap.realm), st->cap.flags, request_perms); - if (in->auth_cap && in->auth_cap->session == session) + if (in->auth_cap && in->auth_cap->session == session) { in->max_size = st->max_size; + in->rstat = st->rstat; + } } else in->snap_caps |= st->cap.caps; @@ -2076,6 +2077,10 @@ void Client::handle_client_session(MClientSession *m) break; case CEPH_SESSION_STALE: + // invalidate session caps/leases + session->cap_gen++; + session->cap_ttl = ceph_clock_now(); + session->cap_ttl -= 1; renew_caps(session); break; @@ -3232,7 +3237,7 @@ void Client::cap_delay_requeue(Inode *in) ldout(cct, 10) << "cap_delay_requeue on " << *in << dendl; in->hold_caps_until = ceph_clock_now(); in->hold_caps_until += cct->_conf->client_caps_release_delay; - delayed_caps.push_back(&in->cap_item); + delayed_list.push_back(&in->delay_cap_item); } void Client::send_cap(Inode *in, MetaSession *session, Cap *cap, @@ -3762,7 +3767,7 @@ void Client::_invalidate_inode_cache(Inode *in, int64_t off, int64_t len) if (cct->_conf->client_oc) { vector ls; Striper::file_to_extents(cct, in->ino, &in->layout, off, len, in->truncate_size, ls); - objectcacher->discard_set(&in->oset, ls); + objectcacher->discard_writeback(&in->oset, ls, nullptr); } _schedule_invalidate_callback(in, off, len); @@ -3901,7 +3906,6 @@ void Client::add_update_cap(Inode *in, MetaSession *mds_session, uint64_t cap_id cap->session = mds_session; cap->inode = in; cap->gen = mds_session->cap_gen; - cap_list.push_back(&in->cap_item); } check_cap_issue(in, cap, issued); @@ -3925,6 +3929,7 @@ void Client::add_update_cap(Inode *in, MetaSession *mds_session, uint64_t cap_id cap->seq = seq; cap->issue_seq = seq; cap->mseq = mseq; + cap->gen = mds_session->cap_gen; cap->latest_perms = cap_perms; ldout(cct, 10) << "add_update_cap issued " << ccap_string(old_caps) << " -> " << ccap_string(cap->issued) << " from mds." << mds @@ -4020,7 +4025,7 @@ void Client::remove_session_caps(MetaSession *s) in->flushing_cap_tids.clear(); } in->flushing_caps = 0; - in->dirty_caps = 0; + in->mark_caps_clean(); put_inode(in); } } @@ -4085,16 +4090,17 @@ void Client::_invalidate_kernel_dcache() } } -void Client::trim_caps(MetaSession *s, int max) +void Client::trim_caps(MetaSession *s, uint64_t max) { mds_rank_t mds = s->mds_num; - int caps_size = s->caps.size(); + size_t caps_size = s->caps.size(); ldout(cct, 10) << "trim_caps mds." << mds << " max " << max << " caps " << caps_size << dendl; - int trimmed = 0; - xlist::iterator p = s->caps.begin(); - std::set anchor; /* prevent put_inode from deleting all caps during traversal */ + uint64_t trimmed = 0; + auto p = s->caps.begin(); + std::set to_trim; /* this avoids caps other than the one we're + * looking at from getting deleted during traversal. */ while ((caps_size - trimmed) > max && !p.end()) { Cap *cap = *p; InodeRef in(cap->inode); @@ -4109,8 +4115,7 @@ void Client::trim_caps(MetaSession *s, int max) // disposable non-auth cap if (!(get_caps_used(in.get()) & ~oissued & mine)) { ldout(cct, 20) << " removing unused, unneeded non-auth cap on " << *in << dendl; - remove_cap(cap, true); - /* N.B. no need to push onto anchor, as we are only removing one cap */ + cap = (remove_cap(cap, true), nullptr); trimmed++; } } else { @@ -4127,9 +4132,8 @@ void Client::trim_caps(MetaSession *s, int max) // the end of this function. _schedule_invalidate_dentry_callback(dn, true); } - ldout(cct, 20) << " anchoring inode: " << in->ino << dendl; - anchor.insert(in); - trim_dentry(dn); + ldout(cct, 20) << " queueing dentry for trimming: " << dn->name << dendl; + to_trim.insert(dn); } else { ldout(cct, 20) << " not expirable: " << dn->name << dendl; all = false; @@ -4141,8 +4145,11 @@ void Client::trim_caps(MetaSession *s, int max) } } } - ldout(cct, 20) << " clearing anchored inodes" << dendl; - anchor.clear(); + ldout(cct, 20) << " trimming queued dentries: " << dendl; + for (const auto &dn : to_trim) { + trim_dentry(dn); + } + to_trim.clear(); caps_size = s->caps.size(); if (caps_size > max) @@ -4159,15 +4166,6 @@ void Client::force_session_readonly(MetaSession *s) } } -void Client::mark_caps_dirty(Inode *in, int caps) -{ - ldout(cct, 10) << "mark_caps_dirty " << *in << " " << ccap_string(in->dirty_caps) << " -> " - << ccap_string(in->dirty_caps | caps) << dendl; - if (caps && !in->caps_dirty()) - in->get(); - in->dirty_caps |= caps; -} - int Client::mark_caps_flushing(Inode *in, ceph_tid_t* ptid) { MetaSession *session = in->auth_cap->session; @@ -4186,7 +4184,7 @@ int Client::mark_caps_flushing(Inode *in, ceph_tid_t* ptid) } in->flushing_caps |= flushing; - in->dirty_caps = 0; + in->mark_caps_clean(); if (!in->flushing_cap_item.is_on_list()) session->flushing_caps.push_back(&in->flushing_cap_item); @@ -4222,20 +4220,20 @@ void Client::adjust_session_flushing_caps(Inode *in, MetaSession *old_s, MetaSe void Client::flush_caps_sync() { ldout(cct, 10) << __func__ << dendl; - xlist::iterator p = delayed_caps.begin(); + xlist::iterator p = delayed_list.begin(); while (!p.end()) { unsigned flags = CHECK_CAPS_NODELAY; Inode *in = *p; ++p; - delayed_caps.pop_front(); - if (p.end() && cap_list.empty()) + delayed_list.pop_front(); + if (p.end() && dirty_list.empty()) flags |= CHECK_CAPS_SYNCHRONOUS; check_caps(in, flags); } // other caps, too - p = cap_list.begin(); + p = dirty_list.begin(); while (!p.end()) { unsigned flags = CHECK_CAPS_NODELAY; Inode *in = *p; @@ -5041,6 +5039,7 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient << " caps now " << ccap_string(new_caps) << " was " << ccap_string(old_caps) << dendl; cap->seq = m->get_seq(); + cap->gen = session->cap_gen; in->layout = m->get_layout(); @@ -5069,6 +5068,12 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient ::decode(in->xattrs, p); in->xattr_version = m->head.xattr_version; } + + if ((new_caps & CEPH_CAP_FILE_SHARED) && m->dirstat_is_valid()) { + in->dirstat.nfiles = m->get_nfiles(); + in->dirstat.nsubdirs = m->get_nsubdirs(); + } + update_inode_file_bits(in, m->get_truncate_seq(), m->get_truncate_size(), m->get_size(), m->get_change_attr(), m->get_time_warp_seq(), m->get_ctime(), m->get_mtime(), m->get_atime(), @@ -5104,17 +5109,16 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient else if (revoked & ceph_deleg_caps_for_type(CEPH_DELEGATION_WR)) in->recall_deleg(true); - if (((used & ~new_caps) & CEPH_CAP_FILE_BUFFER) - && !_flush(in, new C_Client_FlushComplete(this, in))) { + if ((used & revoked & CEPH_CAP_FILE_BUFFER) && + !_flush(in, new C_Client_FlushComplete(this, in))) { // waitin' for flush - } else if ((old_caps & ~new_caps) & CEPH_CAP_FILE_CACHE) { + } else if (revoked & CEPH_CAP_FILE_CACHE) { if (_release(in)) check = true; } else { cap->wanted = 0; // don't let check_caps skip sending a response to MDS check = true; } - } else if (old_caps == new_caps) { ldout(cct, 10) << " caps unchanged at " << ccap_string(old_caps) << dendl; } else { @@ -5149,63 +5153,6 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient m->put(); } -int Client::_getgrouplist(gid_t** sgids, uid_t uid, gid_t gid) -{ - // cppcheck-suppress variableScope - int sgid_count; - gid_t *sgid_buf; - - if (getgroups_cb) { - sgid_count = getgroups_cb(callback_handle, &sgid_buf); - if (sgid_count > 0) { - *sgids = sgid_buf; - return sgid_count; - } - } - -#if HAVE_GETGROUPLIST - struct passwd *pw; - pw = getpwuid(uid); - if (pw == NULL) { - ldout(cct, 3) << "getting user entry failed" << dendl; - return -errno; - } - //use PAM to get the group list - // initial number of group entries, defaults to posix standard of 16 - // PAM implementations may provide more than 16 groups.... - sgid_count = 16; - sgid_buf = (gid_t*)malloc(sgid_count * sizeof(gid_t)); - if (sgid_buf == NULL) { - ldout(cct, 3) << "allocating group memory failed" << dendl; - return -ENOMEM; - } - - while (1) { -#if defined(__APPLE__) - if (getgrouplist(pw->pw_name, gid, (int*)sgid_buf, &sgid_count) == -1) { -#else - if (getgrouplist(pw->pw_name, gid, sgid_buf, &sgid_count) == -1) { -#endif - // we need to resize the group list and try again - void *_realloc = NULL; - if ((_realloc = realloc(sgid_buf, sgid_count * sizeof(gid_t))) == NULL) { - ldout(cct, 3) << "allocating group memory failed" << dendl; - free(sgid_buf); - return -ENOMEM; - } - sgid_buf = (gid_t*)_realloc; - continue; - } - // list was successfully retrieved - break; - } - *sgids = sgid_buf; - return sgid_count; -#else - return 0; -#endif -} - int Client::inode_permission(Inode *in, const UserPerm& perms, unsigned want) { if (perms.uid() == 0) @@ -6054,14 +6001,13 @@ void Client::tick() } // delayed caps - xlist::iterator p = delayed_caps.begin(); + xlist::iterator p = delayed_list.begin(); while (!p.end()) { Inode *in = *p; ++p; if (in->hold_caps_until > now) break; - delayed_caps.pop_front(); - cap_list.push_back(&in->cap_item); + delayed_list.pop_front(); check_caps(in, CHECK_CAPS_NODELAY); } @@ -6720,11 +6666,11 @@ int Client::_do_setattr(Inode *in, struct ceph_statx *stx, int mask, in->cap_dirtier_uid = perms.uid(); in->cap_dirtier_gid = perms.gid(); if (issued & CEPH_CAP_AUTH_EXCL) - mark_caps_dirty(in, CEPH_CAP_AUTH_EXCL); + in->mark_caps_dirty(CEPH_CAP_AUTH_EXCL); else if (issued & CEPH_CAP_FILE_EXCL) - mark_caps_dirty(in, CEPH_CAP_FILE_EXCL); + in->mark_caps_dirty(CEPH_CAP_FILE_EXCL); else if (issued & CEPH_CAP_XATTR_EXCL) - mark_caps_dirty(in, CEPH_CAP_XATTR_EXCL); + in->mark_caps_dirty(CEPH_CAP_XATTR_EXCL); else mask |= CEPH_SETATTR_CTIME; } @@ -6739,7 +6685,7 @@ int Client::_do_setattr(Inode *in, struct ceph_statx *stx, int mask, in->cap_dirtier_uid = perms.uid(); in->cap_dirtier_gid = perms.gid(); in->uid = stx->stx_uid; - mark_caps_dirty(in, CEPH_CAP_AUTH_EXCL); + in->mark_caps_dirty(CEPH_CAP_AUTH_EXCL); mask &= ~CEPH_SETATTR_UID; kill_sguid = true; ldout(cct,10) << "changing uid to " << stx->stx_uid << dendl; @@ -6749,7 +6695,7 @@ int Client::_do_setattr(Inode *in, struct ceph_statx *stx, int mask, in->cap_dirtier_uid = perms.uid(); in->cap_dirtier_gid = perms.gid(); in->gid = stx->stx_gid; - mark_caps_dirty(in, CEPH_CAP_AUTH_EXCL); + in->mark_caps_dirty(CEPH_CAP_AUTH_EXCL); mask &= ~CEPH_SETATTR_GID; kill_sguid = true; ldout(cct,10) << "changing gid to " << stx->stx_gid << dendl; @@ -6760,13 +6706,13 @@ int Client::_do_setattr(Inode *in, struct ceph_statx *stx, int mask, in->cap_dirtier_uid = perms.uid(); in->cap_dirtier_gid = perms.gid(); in->mode = (in->mode & ~07777) | (stx->stx_mode & 07777); - mark_caps_dirty(in, CEPH_CAP_AUTH_EXCL); + in->mark_caps_dirty(CEPH_CAP_AUTH_EXCL); mask &= ~CEPH_SETATTR_MODE; ldout(cct,10) << "changing mode to " << stx->stx_mode << dendl; } else if (kill_sguid && S_ISREG(in->mode) && (in->mode & (S_IXUSR|S_IXGRP|S_IXOTH))) { /* Must squash the any setuid/setgid bits with an ownership change */ in->mode &= ~(S_ISUID|S_ISGID); - mark_caps_dirty(in, CEPH_CAP_AUTH_EXCL); + in->mark_caps_dirty(CEPH_CAP_AUTH_EXCL); } if (mask & CEPH_SETATTR_BTIME) { @@ -6774,7 +6720,7 @@ int Client::_do_setattr(Inode *in, struct ceph_statx *stx, int mask, in->cap_dirtier_uid = perms.uid(); in->cap_dirtier_gid = perms.gid(); in->btime = utime_t(stx->stx_btime); - mark_caps_dirty(in, CEPH_CAP_AUTH_EXCL); + in->mark_caps_dirty(CEPH_CAP_AUTH_EXCL); mask &= ~CEPH_SETATTR_BTIME; ldout(cct,10) << "changing btime to " << in->btime << dendl; } @@ -6793,7 +6739,7 @@ int Client::_do_setattr(Inode *in, struct ceph_statx *stx, int mask, in->cap_dirtier_uid = perms.uid(); in->cap_dirtier_gid = perms.gid(); in->time_warp_seq++; - mark_caps_dirty(in, CEPH_CAP_FILE_EXCL); + in->mark_caps_dirty(CEPH_CAP_FILE_EXCL); mask &= ~(CEPH_SETATTR_MTIME|CEPH_SETATTR_ATIME); } } @@ -7117,7 +7063,22 @@ int Client::fill_stat(Inode *in, struct stat *st, frag_info_t *dirstat, nest_inf st->st_dev = in->snapid; st->st_mode = in->mode; st->st_rdev = in->rdev; - st->st_nlink = in->nlink; + if (in->is_dir()) { + switch (in->nlink) { + case 0: + st->st_nlink = 0; /* dir is unlinked */ + break; + case 1: + st->st_nlink = 1 /* parent dentry */ + + 1 /* /. */ + + in->dirstat.nsubdirs; /* include /. self-reference */ + break; + default: + ceph_abort(); + } + } else { + st->st_nlink = in->nlink; + } st->st_uid = in->uid; st->st_gid = in->gid; if (in->ctime > in->mtime) { @@ -7184,7 +7145,22 @@ void Client::fill_statx(Inode *in, unsigned int mask, struct ceph_statx *stx) } if (mask & CEPH_CAP_LINK_SHARED) { - stx->stx_nlink = in->nlink; + if (in->is_dir()) { + switch (in->nlink) { + case 0: + stx->stx_nlink = 0; /* dir is unlinked */ + break; + case 1: + stx->stx_nlink = 1 /* parent dentry */ + + 1 /* /. */ + + in->dirstat.nsubdirs; /* include /. self-reference */ + break; + default: + ceph_abort(); + } + } else { + stx->stx_nlink = in->nlink; + } stx->stx_mask |= CEPH_STATX_NLINK; } @@ -7755,6 +7731,7 @@ int Client::_readdir_cache_cb(dir_result_t *dirp, add_dirent_cb_t cb, void *p, else dirp->next_offset = dirp->offset_low(); dirp->last_name = dn_name; // we successfully returned this one; update! + dirp->release_count = 0; // last_name no longer match cache index if (r > 0) return r; } @@ -8872,7 +8849,7 @@ done: if (uninline_ret >= 0 || uninline_ret == -ECANCELED) { in->inline_data.clear(); in->inline_version = CEPH_INLINE_NONE; - mark_caps_dirty(in, CEPH_CAP_FILE_WR); + in->mark_caps_dirty(CEPH_CAP_FILE_WR); check_caps(in, 0); } else r = uninline_ret; @@ -9156,8 +9133,9 @@ int Client::_write(Fh *f, int64_t offset, uint64_t size, const char *buf, // check quota uint64_t endoff = offset + size; - if (endoff > in->size && is_quota_bytes_exceeded(in, endoff - in->size, - f->actor_perms)) { + std::list quota_roots; + if (endoff > in->size && + is_quota_bytes_exceeded(in, endoff - in->size, f->actor_perms, "a_roots)) { return -EDQUOT; } @@ -9332,9 +9310,9 @@ success: // extend file? if (totalwritten + offset > in->size) { in->size = totalwritten + offset; - mark_caps_dirty(in, CEPH_CAP_FILE_WR); + in->mark_caps_dirty(CEPH_CAP_FILE_WR); - if (is_quota_bytes_approaching(in, f->actor_perms)) { + if (is_quota_bytes_approaching(in, quota_roots)) { check_caps(in, CHECK_CAPS_NODELAY); } else if (is_max_size_approaching(in)) { check_caps(in, 0); @@ -9348,7 +9326,7 @@ success: // mtime in->mtime = ceph_clock_now(); in->change_attr++; - mark_caps_dirty(in, CEPH_CAP_FILE_WR); + in->mark_caps_dirty(CEPH_CAP_FILE_WR); done: @@ -9363,7 +9341,7 @@ done: if (uninline_ret >= 0 || uninline_ret == -ECANCELED) { in->inline_data.clear(); in->inline_version = CEPH_INLINE_NONE; - mark_caps_dirty(in, CEPH_CAP_FILE_WR); + in->mark_caps_dirty(CEPH_CAP_FILE_WR); check_caps(in, 0); } else r = uninline_ret; @@ -9478,6 +9456,8 @@ int Client::_fsync(Inode *in, bool syncdataonly) } else ldout(cct, 10) << "no metadata needs to commit" << dendl; if (!syncdataonly && !in->unsafe_ops.empty()) { + flush_mdlog_sync(); + MetaRequest *req = in->unsafe_ops.back(); ldout(cct, 15) << "waiting on unsafe requests, last tid " << req->get_tid() << dendl; @@ -10058,7 +10038,6 @@ void Client::ll_register_callbacks(struct client_callback_args *args) ldout(cct, 10) << "ll_register_callbacks cb " << args->handle << " invalidate_ino_cb " << args->ino_cb << " invalidate_dentry_cb " << args->dentry_cb - << " getgroups_cb" << args->getgroups_cb << " switch_interrupt_cb " << args->switch_intr_cb << " remount_cb " << args->remount_cb << dendl; @@ -10079,7 +10058,6 @@ void Client::ll_register_callbacks(struct client_callback_args *args) remount_cb = args->remount_cb; remount_finisher.start(); } - getgroups_cb = args->getgroups_cb; umask_cb = args->umask_cb; } @@ -10852,7 +10830,11 @@ int Client::_getxattr(Inode *in, const char *name, void *value, size_t size, // Do a force getattr to get the latest quota before returning // a value to userspace. - r = _getattr(in, 0, perms, true); + int flags = 0; + if (vxattr->flags & VXATTR_RSTAT) { + flags |= CEPH_STAT_RSTAT; + } + r = _getattr(in, flags, perms, true); if (r != 0) { // Error from getattr! return r; @@ -11369,6 +11351,16 @@ size_t Client::_vxattrcb_dir_rctime(Inode *in, char *val, size_t size) readonly: true, \ hidden: false, \ exists_cb: NULL, \ + flags: 0, \ +} +#define XATTR_NAME_CEPH2(_type, _name, _flags) \ +{ \ + name: CEPH_XATTR_NAME(_type, _name), \ + getxattr_cb: &Client::_vxattrcb_ ## _type ## _ ## _name, \ + readonly: true, \ + hidden: false, \ + exists_cb: NULL, \ + flags: _flags, \ } #define XATTR_LAYOUT_FIELD(_type, _name, _field) \ { \ @@ -11377,6 +11369,7 @@ size_t Client::_vxattrcb_dir_rctime(Inode *in, char *val, size_t size) readonly: false, \ hidden: true, \ exists_cb: &Client::_vxattrcb_layout_exists, \ + flags: 0, \ } #define XATTR_QUOTA_FIELD(_type, _name) \ { \ @@ -11385,6 +11378,7 @@ size_t Client::_vxattrcb_dir_rctime(Inode *in, char *val, size_t size) readonly: false, \ hidden: true, \ exists_cb: &Client::_vxattrcb_quota_exists, \ + flags: 0, \ } const Client::VXattr Client::_dir_vxattrs[] = { @@ -11394,6 +11388,7 @@ const Client::VXattr Client::_dir_vxattrs[] = { readonly: false, hidden: true, exists_cb: &Client::_vxattrcb_layout_exists, + flags: 0, }, XATTR_LAYOUT_FIELD(dir, layout, stripe_unit), XATTR_LAYOUT_FIELD(dir, layout, stripe_count), @@ -11403,17 +11398,18 @@ const Client::VXattr Client::_dir_vxattrs[] = { XATTR_NAME_CEPH(dir, entries), XATTR_NAME_CEPH(dir, files), XATTR_NAME_CEPH(dir, subdirs), - XATTR_NAME_CEPH(dir, rentries), - XATTR_NAME_CEPH(dir, rfiles), - XATTR_NAME_CEPH(dir, rsubdirs), - XATTR_NAME_CEPH(dir, rbytes), - XATTR_NAME_CEPH(dir, rctime), + XATTR_NAME_CEPH2(dir, rentries, VXATTR_RSTAT), + XATTR_NAME_CEPH2(dir, rfiles, VXATTR_RSTAT), + XATTR_NAME_CEPH2(dir, rsubdirs, VXATTR_RSTAT), + XATTR_NAME_CEPH2(dir, rbytes, VXATTR_RSTAT), + XATTR_NAME_CEPH2(dir, rctime, VXATTR_RSTAT), { name: "ceph.quota", getxattr_cb: &Client::_vxattrcb_quota, readonly: false, hidden: true, exists_cb: &Client::_vxattrcb_quota_exists, + flags: 0, }, XATTR_QUOTA_FIELD(quota, max_bytes), XATTR_QUOTA_FIELD(quota, max_files), @@ -11427,6 +11423,7 @@ const Client::VXattr Client::_file_vxattrs[] = { readonly: false, hidden: true, exists_cb: &Client::_vxattrcb_layout_exists, + flags: 0, }, XATTR_LAYOUT_FIELD(file, layout, stripe_unit), XATTR_LAYOUT_FIELD(file, layout, stripe_count), @@ -12395,7 +12392,7 @@ int Client::ll_get_stripe_osd(Inode *in, uint64_t blockno, { Mutex::Locker lock(client_lock); - inodeno_t ino = ll_get_inodeno(in); + inodeno_t ino = in->ino; uint32_t object_size = layout->object_size; uint32_t su = layout->stripe_unit; uint32_t stripe_count = layout->stripe_count; @@ -12879,6 +12876,19 @@ int Client::ll_fsync(Fh *fh, bool syncdataonly) return r; } +int Client::ll_sync_inode(Inode *in, bool syncdataonly) +{ + Mutex::Locker lock(client_lock); + ldout(cct, 3) << "ll_sync_inode " << *in << " " << dendl; + tout(cct) << "ll_sync_inode" << std::endl; + tout(cct) << (unsigned long)in << std::endl; + + if (unmounting) + return -ENOTCONN; + + return _fsync(in, syncdataonly); +} + #ifdef FALLOC_FL_PUNCH_HOLE int Client::_fallocate(Fh *fh, int mode, int64_t offset, int64_t length) @@ -12906,9 +12916,10 @@ int Client::_fallocate(Fh *fh, int mode, int64_t offset, int64_t length) return -EBADF; uint64_t size = offset + length; + std::list quota_roots; if (!(mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE)) && size > in->size && - is_quota_bytes_exceeded(in, size - in->size, fh->actor_perms)) { + is_quota_bytes_exceeded(in, size - in->size, fh->actor_perms, "a_roots)) { return -EDQUOT; } @@ -12943,7 +12954,7 @@ int Client::_fallocate(Fh *fh, int mode, int64_t offset, int64_t length) } in->mtime = ceph_clock_now(); in->change_attr++; - mark_caps_dirty(in, CEPH_CAP_FILE_WR); + in->mark_caps_dirty(CEPH_CAP_FILE_WR); } else { if (in->inline_version < CEPH_INLINE_NONE) { onuninline = new C_SafeCond(&uninline_flock, @@ -12969,7 +12980,7 @@ int Client::_fallocate(Fh *fh, int mode, int64_t offset, int64_t length) 0, true, onfinish); in->mtime = ceph_clock_now(); in->change_attr++; - mark_caps_dirty(in, CEPH_CAP_FILE_WR); + in->mark_caps_dirty(CEPH_CAP_FILE_WR); client_lock.Unlock(); flock.Lock(); @@ -12985,9 +12996,9 @@ int Client::_fallocate(Fh *fh, int mode, int64_t offset, int64_t length) in->size = size; in->mtime = ceph_clock_now(); in->change_attr++; - mark_caps_dirty(in, CEPH_CAP_FILE_WR); + in->mark_caps_dirty(CEPH_CAP_FILE_WR); - if (is_quota_bytes_approaching(in, fh->actor_perms)) { + if (is_quota_bytes_approaching(in, quota_roots)) { check_caps(in, CHECK_CAPS_NODELAY); } else if (is_max_size_approaching(in)) { check_caps(in, 0); @@ -13006,7 +13017,7 @@ int Client::_fallocate(Fh *fh, int mode, int64_t offset, int64_t length) if (uninline_ret >= 0 || uninline_ret == -ECANCELED) { in->inline_data.clear(); in->inline_version = CEPH_INLINE_NONE; - mark_caps_dirty(in, CEPH_CAP_FILE_WR); + in->mark_caps_dirty(CEPH_CAP_FILE_WR); check_caps(in, 0); } else r = uninline_ret; @@ -13473,6 +13484,7 @@ void Client::ms_handle_remote_reset(Connection *con) case MetaSession::STATE_OPEN: { + objecter->maybe_request_map(); /* to check if we are blacklisted */ const md_config_t *conf = cct->_conf; if (conf->client_reconnect_stale) { ldout(cct, 1) << "reset from mds we were open; close mds session for reconnect" << dendl; @@ -13615,32 +13627,34 @@ bool Client::is_quota_files_exceeded(Inode *in, const UserPerm& perms) } bool Client::is_quota_bytes_exceeded(Inode *in, int64_t new_bytes, - const UserPerm& perms) + const UserPerm& perms, + std::list* quota_roots) { return check_quota_condition(in, perms, - [&new_bytes](const Inode &in) { + [&new_bytes, quota_roots](const Inode &in) { + if (quota_roots) + quota_roots->emplace_back(const_cast(&in)); return in.quota.max_bytes && (in.rstat.rbytes + new_bytes) > in.quota.max_bytes; }); } -bool Client::is_quota_bytes_approaching(Inode *in, const UserPerm& perms) +bool Client::is_quota_bytes_approaching(Inode *in, std::list& quota_roots) { - return check_quota_condition(in, perms, - [](const Inode &in) { - if (in.quota.max_bytes) { - if (in.rstat.rbytes >= in.quota.max_bytes) { - return true; - } - - assert(in.size >= in.reported_size); - const uint64_t space = in.quota.max_bytes - in.rstat.rbytes; - const uint64_t size = in.size - in.reported_size; - return (space >> 4) < size; - } else { - return false; - } - }); + assert(in->size >= in->reported_size); + const uint64_t size = in->size - in->reported_size; + + for (auto& diri : quota_roots) { + if (diri->quota.max_bytes) { + if (diri->rstat.rbytes >= diri->quota.max_bytes) + return true; + + uint64_t space = diri->quota.max_bytes - diri->rstat.rbytes; + if ((space >> 4) < size) + return true; + } + } + return false; } enum { @@ -13895,13 +13909,6 @@ void Client::handle_conf_change(const struct md_config_t *conf, } } -void Client::init_groups(UserPerm *perms) -{ - gid_t *sgids; - int count = _getgrouplist(&sgids, perms->uid(), perms->gid()); - perms->init_gids(sgids, count); -} - void intrusive_ptr_add_ref(Inode *in) { in->get(); diff --git a/ceph/src/client/Client.h b/ceph/src/client/Client.h index 0d7cd787b..9c076214a 100644 --- a/ceph/src/client/Client.h +++ b/ceph/src/client/Client.h @@ -94,7 +94,8 @@ class MDSCommandOp : public CommandOp }; /* error code for ceph_fuse */ -#define CEPH_FUSE_NO_MDS_UP -(1<<2) /* no mds up deteced in ceph_fuse */ +#define CEPH_FUSE_NO_MDS_UP -((1<<16)+0) /* no mds up deteced in ceph_fuse */ +#define CEPH_FUSE_LAST -((1<<16)+1) /* (unused) */ // ============================================ // types for my local metadata cache @@ -135,7 +136,6 @@ typedef void (*client_dentry_callback_t)(void *handle, vinodeno_t dirino, vinodeno_t ino, string& name); typedef int (*client_remount_callback_t)(void *handle); -typedef int (*client_getgroups_callback_t)(void *handle, gid_t **sgids); typedef void(*client_switch_interrupt_callback_t)(void *handle, void *data); typedef mode_t (*client_umask_callback_t)(void *handle); @@ -148,7 +148,6 @@ struct client_callback_args { client_dentry_callback_t dentry_cb; client_switch_interrupt_callback_t switch_intr_cb; client_remount_callback_t remount_cb; - client_getgroups_callback_t getgroups_cb; client_umask_callback_t umask_cb; }; @@ -276,7 +275,6 @@ class Client : public Dispatcher, public md_config_obs_t { client_remount_callback_t remount_cb; client_ino_callback_t ino_invalidate_cb; client_dentry_callback_t dentry_invalidate_cb; - client_getgroups_callback_t getgroups_cb; client_umask_callback_t umask_cb; bool can_invalidate_dentries; @@ -432,8 +430,8 @@ protected: Inode* root_ancestor; LRU lru; // lru list of Dentry's in our local metadata cache. - // all inodes with caps sit on either cap_list or delayed_caps. - xlist delayed_caps, cap_list; + // dirty_list keeps all the dirty inodes before flushing. + xlist delayed_list, dirty_list; int num_flushing_caps; ceph::unordered_map snap_realms; @@ -539,7 +537,7 @@ protected: void trim_cache(bool trim_kernel_dcache=false); void trim_cache_for_reconnect(MetaSession *s); void trim_dentry(Dentry *dn); - void trim_caps(MetaSession *s, int max); + void trim_caps(MetaSession *s, uint64_t max); void _invalidate_kernel_dcache(); void dump_inode(Formatter *f, Inode *in, set& did, bool disconnected); @@ -574,8 +572,9 @@ protected: std::function test); bool is_quota_files_exceeded(Inode *in, const UserPerm& perms); bool is_quota_bytes_exceeded(Inode *in, int64_t new_bytes, - const UserPerm& perms); - bool is_quota_bytes_approaching(Inode *in, const UserPerm& perms); + const UserPerm& perms, + std::list* quota_roots=nullptr); + bool is_quota_bytes_approaching(Inode *in, std::list& quota_roots); std::map, int> pool_perms; list waiting_for_pool_perm; @@ -635,7 +634,6 @@ protected: void remove_cap(Cap *cap, bool queue_release); void remove_all_caps(Inode *in); void remove_session_caps(MetaSession *session); - void mark_caps_dirty(Inode *in, int caps); int mark_caps_flushing(Inode *in, ceph_tid_t *ptid); void adjust_session_flushing_caps(Inode *in, MetaSession *old_s, MetaSession *new_s); void flush_caps_sync(); @@ -733,6 +731,7 @@ protected: void flush_mdlog_sync(); void flush_mdlog(MetaSession *session); + xlist &get_dirty_list() { return dirty_list; } // ---------------------- // fs ops. private: @@ -863,9 +862,6 @@ private: MAY_READ = 4, }; - void init_groups(UserPerm *groups); - - int inode_permission(Inode *in, const UserPerm& perms, unsigned want); int xattr_permission(Inode *in, const char *name, unsigned want, const UserPerm& perms); int may_setattr(Inode *in, struct ceph_statx *stx, int mask, @@ -877,7 +873,6 @@ private: int may_hardlink(Inode *in, const UserPerm& perms); int _getattr_for_perm(Inode *in, const UserPerm& perms); - int _getgrouplist(gid_t **sgids, uid_t uid, gid_t gid); vinodeno_t _get_vino(Inode *in); inodeno_t _get_inodeno(Inode *in); @@ -891,8 +886,12 @@ private: size_t (Client::*getxattr_cb)(Inode *in, char *val, size_t size); bool readonly, hidden; bool (Client::*exists_cb)(Inode *in); + int flags; }; +/* Flags for VXattr */ +#define VXATTR_RSTAT 0x1 + bool _vxattrcb_quota_exists(Inode *in); size_t _vxattrcb_quota(Inode *in, char *val, size_t size); size_t _vxattrcb_quota_max_bytes(Inode *in, char *val, size_t size); @@ -1126,15 +1125,13 @@ public: int mksnap(const char *path, const char *name, const UserPerm& perm); int rmsnap(const char *path, const char *name, const UserPerm& perm); + // Inode permission checking + int inode_permission(Inode *in, const UserPerm& perms, unsigned want); + // expose caps int get_caps_issued(int fd); int get_caps_issued(const char *path, const UserPerm& perms); - // low-level interface v2 - inodeno_t ll_get_inodeno(Inode *in) { - Mutex::Locker lock(client_lock); - return _get_inodeno(in); - } snapid_t ll_get_snapid(Inode *in); vinodeno_t ll_get_vino(Inode *in) { Mutex::Locker lock(client_lock); @@ -1221,6 +1218,7 @@ public: loff_t ll_lseek(Fh *fh, loff_t offset, int whence); int ll_flush(Fh *fh); int ll_fsync(Fh *fh, bool syncdataonly); + int ll_sync_inode(Inode *in, bool syncdataonly); int ll_fallocate(Fh *fh, int mode, loff_t offset, loff_t length); int ll_release(Fh *fh); int ll_getlk(Fh *fh, struct flock *fl, uint64_t owner); diff --git a/ceph/src/client/Inode.cc b/ceph/src/client/Inode.cc index 2fadb2b5b..be898cbd8 100644 --- a/ceph/src/client/Inode.cc +++ b/ceph/src/client/Inode.cc @@ -14,7 +14,8 @@ Inode::~Inode() { - cap_item.remove_myself(); + delay_cap_item.remove_myself(); + dirty_cap_item.remove_myself(); snaprealm_item.remove_myself(); if (snapdir_parent) { @@ -48,6 +49,7 @@ ostream& operator<<(ostream &out, const Inode &in) << " open=" << in.open_by_mode << " mode=" << oct << in.mode << dec << " size=" << in.size << "/" << in.max_size + << " nlink=" << in.nlink << " mtime=" << in.mtime << " caps=" << ccap_string(in.caps_issued()); if (!in.caps.empty()) { @@ -747,3 +749,32 @@ void Inode::unset_deleg(Fh *fh) } } } + +/** +* mark_caps_dirty - mark some caps dirty +* @caps: the dirty caps +* +* note that if there is no dirty and flushing caps before, we need to pin this inode. +* it will be unpined by handle_cap_flush_ack when there are no dirty and flushing caps. +*/ +void Inode::mark_caps_dirty(int caps) +{ + lsubdout(client->cct, client, 10) << __func__ << " " << *this << " " << ccap_string(dirty_caps) << " -> " + << ccap_string(dirty_caps | caps) << dendl; + if (caps && !caps_dirty()) + get(); + dirty_caps |= caps; + client->get_dirty_list().push_back(&dirty_cap_item); +} + +/** +* mark_caps_clean - only clean the dirty_caps and caller should start flushing the dirty caps. +*/ +void Inode::mark_caps_clean() +{ + lsubdout(client->cct, client, 10) << __func__ << " " << *this << dendl; + dirty_caps = 0; + dirty_cap_item.remove_myself(); +} + + diff --git a/ceph/src/client/Inode.h b/ceph/src/client/Inode.h index 5242630d5..614d84a30 100644 --- a/ceph/src/client/Inode.h +++ b/ceph/src/client/Inode.h @@ -177,7 +177,7 @@ struct Inode { int shared_gen, cache_gen; int snap_caps, snap_cap_refs; utime_t hold_caps_until; - xlist::item cap_item, flushing_cap_item; + xlist::item delay_cap_item, dirty_cap_item, flushing_cap_item; SnapRealm *snaprealm; xlist::item snaprealm_item; @@ -247,7 +247,7 @@ struct Inode { cap_dirtier_uid(-1), cap_dirtier_gid(-1), dirty_caps(0), flushing_caps(0), shared_gen(0), cache_gen(0), snap_caps(0), snap_cap_refs(0), - cap_item(this), flushing_cap_item(this), + delay_cap_item(this), dirty_cap_item(this), flushing_cap_item(this), snaprealm(0), snaprealm_item(this), oset((void *)this, newlayout->pool_id, this->ino), reported_size(0), wanted_max_size(0), requested_max_size(0), @@ -306,6 +306,8 @@ struct Inode { int set_deleg(Fh *fh, unsigned type, ceph_deleg_cb_t cb, void *priv); void unset_deleg(Fh *fh); + void mark_caps_dirty(int caps); + void mark_caps_clean(); private: // how many opens for write on this Inode? long open_count_for_write() diff --git a/ceph/src/client/MetaRequest.h b/ceph/src/client/MetaRequest.h index 56dc4d973..524390413 100644 --- a/ceph/src/client/MetaRequest.h +++ b/ceph/src/client/MetaRequest.h @@ -199,7 +199,7 @@ public: } bool auth_is_best() { if ((head.op & CEPH_MDS_OP_WRITE) || head.op == CEPH_MDS_OP_OPEN || - head.op == CEPH_MDS_OP_READDIR) + head.op == CEPH_MDS_OP_READDIR || send_to_auth) return true; return false; } diff --git a/ceph/src/client/UserPerm.h b/ceph/src/client/UserPerm.h index a2d6ccbe8..b73814ffa 100644 --- a/ceph/src/client/UserPerm.h +++ b/ceph/src/client/UserPerm.h @@ -31,7 +31,7 @@ private: m_uid = b.m_uid; m_gid = b.m_gid; gid_count = b.gid_count; - if (gid_count) { + if (gid_count > 0) { gids = new gid_t[gid_count]; alloced_gids = true; for (int i = 0; i < gid_count; ++i) { @@ -79,8 +79,8 @@ public: void init_gids(gid_t* _gids, int count) { gids = _gids; gid_count = count; + alloced_gids = true; } - void take_gids() { alloced_gids = true; } void shallow_copy(const UserPerm& o) { m_uid = o.m_uid; m_gid = o.m_gid; diff --git a/ceph/src/client/fuse_ll.cc b/ceph/src/client/fuse_ll.cc index d24ad5c34..018653f7e 100644 --- a/ceph/src/client/fuse_ll.cc +++ b/ceph/src/client/fuse_ll.cc @@ -127,20 +127,20 @@ static int getgroups(fuse_req_t req, gid_t **sgids) return -ENOSYS; } -static int getgroups_cb(void *handle, gid_t **sgids) +static void get_fuse_groups(UserPerm& perms, fuse_req_t req) { - CephFuse::Handle *cfuse = (CephFuse::Handle *) handle; - fuse_req_t req = cfuse->get_fuse_req(); - return getgroups(req, sgids); -} + if (g_conf->get_val("fuse_set_user_groups")) { + gid_t *gids = NULL; + int count = getgroups(req, &gids); -#define GET_GROUPS(perms, req) { \ - if (g_conf->get_val("fuse_set_user_groups")) { \ - gid_t *gids = NULL; \ - int count = getgroups(req, &gids); \ - perms.init_gids(gids, count); \ - perms.take_gids(); \ - } } + if (count > 0) { + perms.init_gids(gids, count); + } else if (count < 0) { + derr << __func__ << ": getgroups failed: " << cpp_strerror(-count) + << dendl; + } + } +} static CephFuse::Handle *fuse_ll_req_prepare(fuse_req_t req) @@ -158,7 +158,7 @@ static void fuse_ll_lookup(fuse_req_t req, fuse_ino_t parent, const char *name) Inode *i2, *i1 = cfuse->iget(parent); // see below int r; UserPerm perms(ctx->uid, ctx->gid); - GET_GROUPS(perms, req); + get_fuse_groups(perms, req); memset(&fe, 0, sizeof(fe)); r = cfuse->client->ll_lookup(i1, name, &fe.attr, &i2, perms); @@ -191,7 +191,7 @@ static void fuse_ll_getattr(fuse_req_t req, fuse_ino_t ino, Inode *in = cfuse->iget(ino); struct stat stbuf; UserPerm perms(ctx->uid, ctx->gid); - GET_GROUPS(perms, req); + get_fuse_groups(perms, req); (void) fi; // XXX @@ -213,7 +213,7 @@ static void fuse_ll_setattr(fuse_req_t req, fuse_ino_t ino, struct stat *attr, const struct fuse_ctx *ctx = fuse_req_ctx(req); Inode *in = cfuse->iget(ino); UserPerm perms(ctx->uid, ctx->gid); - GET_GROUPS(perms, req); + get_fuse_groups(perms, req); int mask = 0; if (to_set & FUSE_SET_ATTR_MODE) mask |= CEPH_SETATTR_MODE; @@ -250,7 +250,7 @@ static void fuse_ll_setxattr(fuse_req_t req, fuse_ino_t ino, const char *name, const struct fuse_ctx *ctx = fuse_req_ctx(req); Inode *in = cfuse->iget(ino); UserPerm perms(ctx->uid, ctx->gid); - GET_GROUPS(perms, req); + get_fuse_groups(perms, req); int r = cfuse->client->ll_setxattr(in, name, value, size, flags, perms); fuse_reply_err(req, -r); @@ -265,7 +265,7 @@ static void fuse_ll_listxattr(fuse_req_t req, fuse_ino_t ino, size_t size) Inode *in = cfuse->iget(ino); char buf[size]; UserPerm perms(ctx->uid, ctx->gid); - GET_GROUPS(perms, req); + get_fuse_groups(perms, req); int r = cfuse->client->ll_listxattr(in, buf, size, perms); if (size == 0 && r >= 0) @@ -290,7 +290,7 @@ static void fuse_ll_getxattr(fuse_req_t req, fuse_ino_t ino, const char *name, Inode *in = cfuse->iget(ino); char buf[size]; UserPerm perms(ctx->uid, ctx->gid); - GET_GROUPS(perms, req); + get_fuse_groups(perms, req); int r = cfuse->client->ll_getxattr(in, name, buf, size, perms); if (size == 0 && r >= 0) @@ -310,7 +310,7 @@ static void fuse_ll_removexattr(fuse_req_t req, fuse_ino_t ino, const struct fuse_ctx *ctx = fuse_req_ctx(req); Inode *in = cfuse->iget(ino); UserPerm perms(ctx->uid, ctx->gid); - GET_GROUPS(perms, req); + get_fuse_groups(perms, req); int r = cfuse->client->ll_removexattr(in, name, perms); fuse_reply_err(req, -r); @@ -327,7 +327,7 @@ static void fuse_ll_opendir(fuse_req_t req, fuse_ino_t ino, void *dirp; UserPerm perms(ctx->uid, ctx->gid); - GET_GROUPS(perms, req); + get_fuse_groups(perms, req); int r = cfuse->client->ll_opendir(in, fi->flags, (dir_result_t **)&dirp, perms); @@ -348,7 +348,7 @@ static void fuse_ll_readlink(fuse_req_t req, fuse_ino_t ino) Inode *in = cfuse->iget(ino); char buf[PATH_MAX + 1]; // leave room for a null terminator UserPerm perms(ctx->uid, ctx->gid); - GET_GROUPS(perms, req); + get_fuse_groups(perms, req); int r = cfuse->client->ll_readlink(in, buf, sizeof(buf) - 1, perms); if (r >= 0) { @@ -369,7 +369,7 @@ static void fuse_ll_mknod(fuse_req_t req, fuse_ino_t parent, const char *name, Inode *i2, *i1 = cfuse->iget(parent); struct fuse_entry_param fe; UserPerm perms(ctx->uid, ctx->gid); - GET_GROUPS(perms, req); + get_fuse_groups(perms, req); memset(&fe, 0, sizeof(fe)); @@ -398,7 +398,7 @@ static void fuse_ll_mkdir(fuse_req_t req, fuse_ino_t parent, const char *name, memset(&fe, 0, sizeof(fe)); UserPerm perm(ctx->uid, ctx->gid); - GET_GROUPS(perm, req); + get_fuse_groups(perm, req); #ifdef HAVE_SYS_SYNCFS if (cfuse->fino_snap(parent) == CEPH_SNAPDIR && cfuse->client->cct->_conf->fuse_multithreaded && @@ -441,7 +441,7 @@ static void fuse_ll_unlink(fuse_req_t req, fuse_ino_t parent, const char *name) const struct fuse_ctx *ctx = fuse_req_ctx(req); Inode *in = cfuse->iget(parent); UserPerm perm(ctx->uid, ctx->gid); - GET_GROUPS(perm, req); + get_fuse_groups(perm, req); int r = cfuse->client->ll_unlink(in, name, perm); fuse_reply_err(req, -r); @@ -455,7 +455,7 @@ static void fuse_ll_rmdir(fuse_req_t req, fuse_ino_t parent, const char *name) const struct fuse_ctx *ctx = fuse_req_ctx(req); Inode *in = cfuse->iget(parent); UserPerm perms(ctx->uid, ctx->gid); - GET_GROUPS(perms, req); + get_fuse_groups(perms, req); int r = cfuse->client->ll_rmdir(in, name, perms); fuse_reply_err(req, -r); @@ -471,7 +471,7 @@ static void fuse_ll_symlink(fuse_req_t req, const char *existing, Inode *i2, *i1 = cfuse->iget(parent); struct fuse_entry_param fe; UserPerm perms(ctx->uid, ctx->gid); - GET_GROUPS(perms, req); + get_fuse_groups(perms, req); memset(&fe, 0, sizeof(fe)); @@ -497,7 +497,7 @@ static void fuse_ll_rename(fuse_req_t req, fuse_ino_t parent, const char *name, Inode *in = cfuse->iget(parent); Inode *nin = cfuse->iget(newparent); UserPerm perm(ctx->uid, ctx->gid); - GET_GROUPS(perm, req); + get_fuse_groups(perm, req); int r = cfuse->client->ll_rename(in, name, nin, newname, perm); fuse_reply_err(req, -r); @@ -517,7 +517,7 @@ static void fuse_ll_link(fuse_req_t req, fuse_ino_t ino, fuse_ino_t newparent, memset(&fe, 0, sizeof(fe)); UserPerm perm(ctx->uid, ctx->gid); - GET_GROUPS(perm, req); + get_fuse_groups(perm, req); /* * Note that we could successfully link, but then fail the subsequent @@ -556,7 +556,7 @@ static void fuse_ll_open(fuse_req_t req, fuse_ino_t ino, Inode *in = cfuse->iget(ino); Fh *fh = NULL; UserPerm perms(ctx->uid, ctx->gid); - GET_GROUPS(perms, req); + get_fuse_groups(perms, req); int r = cfuse->client->ll_open(in, fi->flags, &fh, perms); if (r == 0) { @@ -748,7 +748,15 @@ static void fuse_ll_fsyncdir(fuse_req_t req, fuse_ino_t ino, int datasync, static void fuse_ll_access(fuse_req_t req, fuse_ino_t ino, int mask) { - fuse_reply_err(req, 0); + CephFuse::Handle *cfuse = fuse_ll_req_prepare(req); + const struct fuse_ctx *ctx = fuse_req_ctx(req); + Inode *in = cfuse->iget(ino); + UserPerm perms(ctx->uid, ctx->gid); + get_fuse_groups(perms, req); + + int r = cfuse->client->inode_permission(in, perms, mask); + fuse_reply_err(req, -r); + cfuse->iput(in); } static void fuse_ll_create(fuse_req_t req, fuse_ino_t parent, const char *name, @@ -760,7 +768,7 @@ static void fuse_ll_create(fuse_req_t req, fuse_ino_t parent, const char *name, struct fuse_entry_param fe; Fh *fh = NULL; UserPerm perms(ctx->uid, ctx->gid); - GET_GROUPS(perms, req); + get_fuse_groups(perms, req); memset(&fe, 0, sizeof(fe)); @@ -791,7 +799,7 @@ static void fuse_ll_statfs(fuse_req_t req, fuse_ino_t ino) Inode *in = cfuse->iget(ino); const struct fuse_ctx *ctx = fuse_req_ctx(req); UserPerm perms(ctx->uid, ctx->gid); - GET_GROUPS(perms, req); + get_fuse_groups(perms, req); int r = cfuse->client->ll_statfs(in, &stbuf, perms); if (r == 0) @@ -1130,7 +1138,6 @@ int CephFuse::Handle::start() #if defined(__linux__) remount_cb: remount_cb, #endif - getgroups_cb: getgroups_cb, #if !defined(DARWIN) umask_cb: umask_cb, #endif diff --git a/ceph/src/cls/rgw/cls_rgw.cc b/ceph/src/cls/rgw/cls_rgw.cc index dc9da5968..d7deb78f8 100644 --- a/ceph/src/cls/rgw/cls_rgw.cc +++ b/ceph/src/cls/rgw/cls_rgw.cc @@ -2296,6 +2296,9 @@ static int list_plain_entries(cls_method_context_t hctx, const string& name, con for (iter = keys.begin(); iter != keys.end(); ++iter) { if (iter->first >= end_key) { /* past the end of plain namespace */ + if (pmore) { + *pmore = false; + } return count; } @@ -2317,6 +2320,10 @@ static int list_plain_entries(cls_method_context_t hctx, const string& name, con CLS_LOG(20, "%s(): entry.idx=%s e.key.name=%s", __func__, escape_str(entry.idx).c_str(), escape_str(e.key.name).c_str()); if (!name.empty() && e.key.name != name) { + /* we are skipping the rest of the entries */ + if (pmore) { + *pmore = false; + } return count; } @@ -2379,6 +2386,10 @@ static int list_instance_entries(cls_method_context_t hctx, const string& name, entry.data = iter->second; if (!filter.empty() && entry.idx.compare(0, filter.size(), filter) != 0) { + /* we are skipping the rest of the entries */ + if (pmore) { + *pmore = false; + } return count; } @@ -2395,6 +2406,10 @@ static int list_instance_entries(cls_method_context_t hctx, const string& name, } if (!name.empty() && e.key.name != name) { + /* we are skipping the rest of the entries */ + if (pmore) { + *pmore = false; + } return count; } @@ -2456,6 +2471,10 @@ static int list_olh_entries(cls_method_context_t hctx, const string& name, const entry.data = iter->second; if (!filter.empty() && entry.idx.compare(0, filter.size(), filter) != 0) { + /* we are skipping the rest of the entries */ + if (pmore) { + *pmore = false; + } return count; } @@ -2472,6 +2491,10 @@ static int list_olh_entries(cls_method_context_t hctx, const string& name, const } if (!name.empty() && e.key.name != name) { + /* we are skipping the rest of the entries */ + if (pmore) { + *pmore = false; + } return count; } @@ -2502,7 +2525,7 @@ static int rgw_bi_list_op(cls_method_context_t hctx, bufferlist *in, bufferlist int32_t max = (op.max < MAX_BI_LIST_ENTRIES ? op.max : MAX_BI_LIST_ENTRIES); string start_key = op.marker; bool more; - int ret = list_plain_entries(hctx, op.name, op.marker, max, &op_ret.entries, &more); + int ret = list_plain_entries(hctx, op.name, op.marker, max, &op_ret.entries, &more); if (ret < 0) { CLS_LOG(0, "ERROR: %s(): list_plain_entries retured ret=%d", __func__, ret); return ret; diff --git a/ceph/src/common/DecayCounter.cc b/ceph/src/common/DecayCounter.cc index 67a129ccd..26a552d43 100644 --- a/ceph/src/common/DecayCounter.cc +++ b/ceph/src/common/DecayCounter.cc @@ -62,21 +62,23 @@ void DecayCounter::generate_test_instances(list& ls) void DecayCounter::decay(utime_t now, const DecayRate &rate) { - utime_t el = now; - el -= last_decay; + if (now >= last_decay) { + double el = (double)(now - last_decay); + if (el >= 1.0) { + // calculate new value + double newval = (val+delta) * exp(el * rate.k); + if (newval < .01) + newval = 0.0; - if (el.sec() >= 1) { - // calculate new value - double newval = (val+delta) * exp((double)el * rate.k); - if (newval < .01) - newval = 0.0; + // calculate velocity approx + vel += (newval - val) * el; + vel *= exp(el * rate.k); - // calculate velocity approx - vel += (newval - val) * (double)el; - vel *= exp((double)el * rate.k); - - val = newval; - delta = 0; - last_decay = now; + val = newval; + delta = 0; + last_decay = now; + } + } else { + last_decay = now; } } diff --git a/ceph/src/common/DecayCounter.h b/ceph/src/common/DecayCounter.h index e45aed8d2..4addb1ce3 100644 --- a/ceph/src/common/DecayCounter.h +++ b/ceph/src/common/DecayCounter.h @@ -86,15 +86,15 @@ public: return val+delta; } - double get_last() { + double get_last() const { return val; } - double get_last_vel() { + double get_last_vel() const { return vel; } - utime_t get_last_decay() { + utime_t get_last_decay() const { return last_decay; } diff --git a/ceph/src/common/Preforker.h b/ceph/src/common/Preforker.h index 9fe34e5ea..f6671a5d3 100644 --- a/ceph/src/common/Preforker.h +++ b/ceph/src/common/Preforker.h @@ -107,11 +107,8 @@ public: int signal_exit(int r) { if (forked) { - // tell parent. this shouldn't fail, but if it does, pass the - // error back to the parent. - int ret = safe_write(fd[1], &r, sizeof(r)); - if (ret <= 0) - return ret; + /* If we get an error here, it's too late to do anything reasonable about it. */ + (void)safe_write(fd[1], &r, sizeof(r)); } return r; } diff --git a/ceph/src/common/bounded_key_counter.h b/ceph/src/common/bounded_key_counter.h index e5aa52210..0109a26ef 100644 --- a/ceph/src/common/bounded_key_counter.h +++ b/ceph/src/common/bounded_key_counter.h @@ -73,7 +73,11 @@ class BoundedKeyCounter { struct const_pointer_iterator : public map_type::const_iterator { const_pointer_iterator(typename map_type::const_iterator i) : map_type::const_iterator(i) {} - const value_type* operator*() const { + + using value_type = typename map_type::const_iterator::value_type*; + using reference = const typename map_type::const_iterator::value_type*; + + reference operator*() const { return &map_type::const_iterator::operator*(); } }; diff --git a/ceph/src/common/code_environment.cc b/ceph/src/common/code_environment.cc index 7b106d585..0aab05c76 100644 --- a/ceph/src/common/code_environment.cc +++ b/ceph/src/common/code_environment.cc @@ -16,10 +16,14 @@ #include +#include "acconfig.h" + #ifdef HAVE_SYS_PRCTL_H #include #endif +#include + code_environment_t g_code_env = CODE_ENVIRONMENT_UTILITY; extern "C" const char *code_environment_to_str(enum code_environment_t e) diff --git a/ceph/src/common/compat.cc b/ceph/src/common/compat.cc new file mode 100644 index 000000000..18b75874a --- /dev/null +++ b/ceph/src/common/compat.cc @@ -0,0 +1,78 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(__linux__) +#include +#endif + +#include "include/compat.h" +#include "common/safe_io.h" + +// The type-value for a ZFS FS in fstatfs. +#define FS_ZFS_TYPE 0xde + +// On FreeBSD, ZFS fallocate always fails since it is considered impossible to +// reserve space on a COW filesystem. posix_fallocate() returns EINVAL +// Linux in this case already emulates the reservation in glibc +// In which case it is allocated manually, and still that is not a real guarantee +// that a full buffer is allocated on disk, since it could be compressed. +// To prevent this the written buffer needs to be loaded with random data. +int manual_fallocate(int fd, off_t offset, off_t len) { + int r = lseek(fd, offset, SEEK_SET); + if (r == -1) + return errno; + char data[1024*128]; + // TODO: compressing filesystems would require random data + memset(data, 0x42, sizeof(data)); + for (off_t off = 0; off < len; off += sizeof(data)) { + if (off + sizeof(data) > len) + r = safe_write(fd, data, len - off); + else + r = safe_write(fd, data, sizeof(data)); + if (r == -1) { + return errno; + } + } + return 0; +} + +int on_zfs(int basedir_fd) { + struct statfs basefs; + (void)fstatfs(basedir_fd, &basefs); + return (basefs.f_type == FS_ZFS_TYPE); +} + +int ceph_posix_fallocate(int fd, off_t offset, off_t len) { + // Return 0 if oke, otherwise errno > 0 + +#ifdef HAVE_POSIX_FALLOCATE + if (on_zfs(fd)) { + return manual_fallocate(fd, offset, len); + } else { + return posix_fallocate(fd, offset, len); + } +#elif defined(__APPLE__) + fstore_t store; + store.fst_flags = F_ALLOCATECONTIG; + store.fst_posmode = F_PEOFPOSMODE; + store.fst_offset = offset; + store.fst_length = len; + + int ret = fcntl(fd, F_PREALLOCATE, &store); + if (ret == -1) { + ret = errno; + } + return ret; +#else + return manual_fallocate(fd, offset, len); +#endif +} + diff --git a/ceph/src/common/crc32c_aarch64.c b/ceph/src/common/crc32c_aarch64.c index e4fbf23a7..d15736a0c 100644 --- a/ceph/src/common/crc32c_aarch64.c +++ b/ceph/src/common/crc32c_aarch64.c @@ -1,6 +1,7 @@ #include "acconfig.h" #include "include/int_types.h" #include "common/crc32c_aarch64.h" +#include "arch/arm.h" #ifndef HAVE_ARMV8_CRC_CRYPTO_INTRINSICS /* Request crc extension capabilities from the assembler */ @@ -96,6 +97,7 @@ uint32_t ceph_crc32c_aarch64(uint32_t crc, unsigned char const *buffer, unsigned if (buffer) { #ifdef HAVE_ARMV8_CRYPTO + if (ceph_arch_aarch64_pmull) { #ifdef HAVE_ARMV8_CRC_CRYPTO_INTRINSICS /* Calculate reflected crc with PMULL Instruction */ const poly64_t k1 = 0xe417f38a, k2 = 0x8f158014; @@ -183,7 +185,7 @@ uint32_t ceph_crc32c_aarch64(uint32_t crc, unsigned char const *buffer, unsigned if(!(length += 1024)) return crc; - + } #endif /* HAVE_ARMV8_CRYPTO */ while ((length -= sizeof(uint64_t)) >= 0) { CRC32CX(crc, *(uint64_t *)buffer); @@ -203,6 +205,7 @@ uint32_t ceph_crc32c_aarch64(uint32_t crc, unsigned char const *buffer, unsigned CRC32CB(crc, *buffer); } else { #ifdef HAVE_ARMV8_CRYPTO + if (ceph_arch_aarch64_pmull) { #ifdef HAVE_ARMV8_CRC_CRYPTO_INTRINSICS const poly64_t k1 = 0xe417f38a; uint64_t t0; @@ -250,7 +253,7 @@ uint32_t ceph_crc32c_aarch64(uint32_t crc, unsigned char const *buffer, unsigned if(!(length += 1024)) return crc; - + } #endif /* HAVE_ARMV8_CRYPTO */ while ((length -= sizeof(uint64_t)) >= 0) CRC32CX(crc, 0); diff --git a/ceph/src/common/legacy_config_opts.h b/ceph/src/common/legacy_config_opts.h index 0651b2070..e2ac86444 100644 --- a/ceph/src/common/legacy_config_opts.h +++ b/ceph/src/common/legacy_config_opts.h @@ -331,9 +331,12 @@ OPTION(auth_service_required, OPT_STR) // required by daemons of clients OPTION(auth_client_required, OPT_STR) // what clients require of daemons OPTION(auth_supported, OPT_STR) // deprecated; default value for above if they are not defined. OPTION(max_rotating_auth_attempts, OPT_INT) -OPTION(cephx_require_signatures, OPT_BOOL) // If true, don't talk to Cephx partners if they don't support message signing; off by default +OPTION(cephx_require_signatures, OPT_BOOL) OPTION(cephx_cluster_require_signatures, OPT_BOOL) OPTION(cephx_service_require_signatures, OPT_BOOL) +OPTION(cephx_require_version, OPT_INT) +OPTION(cephx_cluster_require_version, OPT_INT) +OPTION(cephx_service_require_version, OPT_INT) OPTION(cephx_sign_messages, OPT_BOOL) // Default to signing session messages if supported OPTION(auth_mon_ticket_ttl, OPT_DOUBLE) OPTION(auth_service_ticket_ttl, OPT_DOUBLE) @@ -769,6 +772,8 @@ OPTION(osd_max_scrubs, OPT_INT) OPTION(osd_scrub_during_recovery, OPT_BOOL) // Allow new scrubs to start while recovery is active on the OSD OPTION(osd_scrub_begin_hour, OPT_INT) OPTION(osd_scrub_end_hour, OPT_INT) +OPTION(osd_scrub_begin_week_day, OPT_INT) +OPTION(osd_scrub_end_week_day, OPT_INT) OPTION(osd_scrub_load_threshold, OPT_FLOAT) OPTION(osd_scrub_min_interval, OPT_FLOAT) // if load is low OPTION(osd_scrub_max_interval, OPT_FLOAT) // regardless of load @@ -782,7 +787,12 @@ OPTION(osd_scrub_auto_repair_num_errors, OPT_U32) // only auto-repair when num OPTION(osd_deep_scrub_interval, OPT_FLOAT) // once a week OPTION(osd_deep_scrub_randomize_ratio, OPT_FLOAT) // scrubs will randomly become deep scrubs at this rate (0.15 -> 15% of scrubs are deep) OPTION(osd_deep_scrub_stride, OPT_INT) +OPTION(osd_deep_scrub_keys, OPT_INT) OPTION(osd_deep_scrub_update_digest_min_age, OPT_INT) // objects must be this old (seconds) before we update the whole-object digest on scrub +OPTION(osd_skip_data_digest, OPT_BOOL) +OPTION(osd_distrust_data_digest, OPT_BOOL) +OPTION(osd_deep_scrub_large_omap_object_key_threshold, OPT_U64) +OPTION(osd_deep_scrub_large_omap_object_value_sum_threshold, OPT_U64) OPTION(osd_class_dir, OPT_STR) // where rados plugins are stored OPTION(osd_open_classes_on_start, OPT_BOOL) OPTION(osd_class_load_list, OPT_STR) // list of object classes allowed to be loaded (allow all: *) @@ -818,7 +828,6 @@ OPTION(osd_debug_drop_ping_probability, OPT_DOUBLE) OPTION(osd_debug_drop_ping_duration, OPT_INT) OPTION(osd_debug_op_order, OPT_BOOL) OPTION(osd_debug_verify_missing_on_start, OPT_BOOL) -OPTION(osd_debug_scrub_chance_rewrite_digest, OPT_U64) OPTION(osd_debug_verify_snaps, OPT_BOOL) OPTION(osd_debug_verify_stray_on_activate, OPT_BOOL) OPTION(osd_debug_skip_full_check_in_backfill_reservation, OPT_BOOL) @@ -828,6 +837,7 @@ OPTION(osd_debug_misdirected_ops, OPT_BOOL) OPTION(osd_debug_skip_full_check_in_recovery, OPT_BOOL) OPTION(osd_debug_random_push_read_error, OPT_DOUBLE) OPTION(osd_debug_verify_cached_snaps, OPT_BOOL) +OPTION(osd_debug_deep_scrub_sleep, OPT_FLOAT) OPTION(osd_enable_op_tracker, OPT_BOOL) // enable/disable OSD op tracking OPTION(osd_num_op_tracker_shard, OPT_U32) // The number of shards for holding the ops OPTION(osd_op_history_size, OPT_U32) // Max number of completed ops to track @@ -1335,6 +1345,7 @@ OPTION(rgw_lc_max_objs, OPT_INT) OPTION(rgw_lc_debug_interval, OPT_INT) // Debug run interval, in seconds OPTION(rgw_script_uri, OPT_STR) // alternative value for SCRIPT_URI if not set in request OPTION(rgw_request_uri, OPT_STR) // alternative value for REQUEST_URI if not set in request +OPTION(rgw_ignore_get_invalid_range, OPT_BOOL) // treat invalid (e.g., negative) range requests as full OPTION(rgw_swift_url, OPT_STR) // the swift url, being published by the internal swift auth OPTION(rgw_swift_url_prefix, OPT_STR) // entry point for which a url is considered a swift url OPTION(rgw_swift_auth_url, OPT_STR) // default URL to go and verify tokens for v1 auth (if not using internal swift auth) @@ -1360,7 +1371,6 @@ OPTION(rgw_keystone_accepted_admin_roles, OPT_STR) // list of roles allowing an OPTION(rgw_keystone_token_cache_size, OPT_INT) // max number of entries in keystone token cache OPTION(rgw_keystone_revocation_interval, OPT_INT) // seconds between tokens revocation check OPTION(rgw_keystone_verify_ssl, OPT_BOOL) // should we try to verify keystone's ssl -OPTION(rgw_keystone_implicit_tenants, OPT_BOOL) // create new users in their own tenants of the same name OPTION(rgw_cross_domain_policy, OPT_STR) OPTION(rgw_healthcheck_disabling_path, OPT_STR) // path that existence causes the healthcheck to respond 503 OPTION(rgw_s3_auth_use_rados, OPT_BOOL) // should we try to use the internal credentials for s3? diff --git a/ceph/src/common/obj_bencher.cc b/ceph/src/common/obj_bencher.cc index 4eaeba347..f80d09cee 100644 --- a/ceph/src/common/obj_bencher.cc +++ b/ceph/src/common/obj_bencher.cc @@ -592,7 +592,7 @@ int ObjBencher::write_bench(int secondsToRun, formatter->dump_format("min_iops", "%d", data.idata.min_iops); formatter->dump_format("average_latency", "%f", data.avg_latency); formatter->dump_format("stddev_latency", "%f", vec_stddev(data.history.latency)); - formatter->dump_format("max_latency:", "%f", data.max_latency); + formatter->dump_format("max_latency", "%f", data.max_latency); formatter->dump_format("min_latency", "%f", data.min_latency); } //write object size/number data for read benchmarks diff --git a/ceph/src/common/options.cc b/ceph/src/common/options.cc index 3ea0af62a..5c83f9527 100644 --- a/ceph/src/common/options.cc +++ b/ceph/src/common/options.cc @@ -1479,14 +1479,26 @@ std::vector