From 94b1876350060563a6ac95339df15f95fd3ebadc Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fabian=20Gr=C3=BCnbichler?= Date: Wed, 25 Apr 2018 10:02:09 +0200 Subject: [PATCH] update sources to v12.2.5 --- Makefile | 2 +- ceph/CMakeLists.txt | 21 +- ceph/COPYING | 2 +- ceph/PendingReleaseNotes | 57 + ceph/README.md | 2 +- ceph/alpine/APKBUILD | 6 +- ceph/ceph.spec | 9 +- ceph/ceph.spec.in | 3 +- ceph/debian/changelog | 6 + ceph/debian/control | 5 +- ceph/debian/copyright | 2 +- ceph/doc/cephfs/upgrading.rst | 48 + ceph/doc/conf.py | 2 +- ceph/doc/dev/ceph-volume/lvm.rst | 11 + ceph/doc/dev/index.rst | 2 +- ceph/doc/install/install-ceph-gateway.rst | 14 +- ceph/doc/install/manual-deployment.rst | 80 +- ceph/doc/man/8/ceph-authtool.rst | 9 +- ceph/doc/man/8/ceph-volume.rst | 29 +- ceph/doc/man/8/radosgw-admin.rst | 6 + ceph/doc/mgr/restful.rst | 67 + ceph/doc/mgr/zabbix.rst | 11 +- .../rados/command/list-inconsistent-obj.json | 82 +- .../rados/command/list-inconsistent-snap.json | 12 +- .../configuration/bluestore-config-ref.rst | 6 +- .../rados/configuration/storage-devices.rst | 2 +- .../rados/operations/bluestore-migration.rst | 292 ++ ceph/doc/rados/operations/index.rst | 1 + ceph/doc/rados/operations/pg-states.rst | 31 +- .../troubleshooting/troubleshooting-pg.rst | 12 +- ceph/doc/radosgw/adminops.rst | 11 + ceph/doc/radosgw/config-ref.rst | 12 + ceph/doc/radosgw/frontends.rst | 97 + ceph/doc/radosgw/index.rst | 1 + ceph/man/conf.py | 2 +- ceph/qa/cephfs/clusters/1-mds-1-client.yaml | 8 + ceph/qa/cephfs/clusters/1-mds-2-client.yaml | 9 + ceph/qa/cephfs/clusters/1-mds-4-client.yaml | 11 + ceph/qa/cephfs/overrides/log-config.yaml | 3 + ceph/qa/cephfs/overrides/osd-asserts.yaml | 5 + ceph/qa/standalone/ceph-helpers.sh | 42 +- ceph/qa/standalone/mon/osd-pool-create.sh | 24 +- ceph/qa/standalone/osd/repro_long_log.sh | 151 + .../qa/standalone/scrub/osd-recovery-scrub.sh | 2 +- ceph/qa/standalone/scrub/osd-scrub-repair.sh | 3108 ++++++++++++++--- ceph/qa/standalone/scrub/osd-scrub-snaps.sh | 393 ++- ceph/qa/standalone/scrub/osd-scrub-test.sh | 113 + .../special/ceph_objectstore_tool.py | 2 +- .../tasks/cephfs_scrub_tests.yaml | 2 + .../fs/basic_functional/tasks/mds-full.yaml | 3 + .../cephfs/clusters/1-mds-1-client.yaml | 1 + .../cephfs/clusters/fixed-3-cephfs.yaml | 1 - .../% => kcephfs/cephfs/overrides/+} | 0 .../kcephfs/cephfs/overrides/debug.yaml | 1 + .../kcephfs/cephfs/overrides/frag_enable.yaml | 1 + .../kcephfs/cephfs/overrides/log-config.yaml | 1 + .../kcephfs/cephfs/overrides/osd-asserts.yaml | 1 + .../cephfs/overrides/whitelist_health.yaml | 1 + .../whitelist_wrongly_marked_down.yaml | 1 + .../clusters/1-mds-2-client.yaml | 1 + .../mixed-clients/clusters/2-clients.yaml | 9 - .../suites/kcephfs/mixed-clients/overrides/+ | 0 .../mixed-clients/overrides/debug.yaml | 1 + .../mixed-clients/overrides/frag_enable.yaml | 1 + .../mixed-clients/overrides/log-config.yaml | 1 + .../mixed-clients/overrides/osd-asserts.yaml | 1 + .../overrides/whitelist_health.yaml | 1 + .../whitelist_wrongly_marked_down.yaml | 1 + .../recovery/clusters/1-mds-4-client.yaml | 1 + .../recovery/clusters/4-remote-clients.yaml | 12 - ceph/qa/suites/kcephfs/recovery/overrides/+ | 0 .../kcephfs/recovery/overrides/debug.yaml | 1 + .../recovery/overrides/frag_enable.yaml | 1 + .../recovery/overrides/log-config.yaml | 1 + .../recovery/overrides/osd-asserts.yaml | 1 + .../recovery/overrides/whitelist_health.yaml | 1 + .../whitelist_wrongly_marked_down.yaml | 1 + .../kcephfs/recovery/tasks/mds-full.yaml | 12 +- .../thrash/clusters/1-mds-1-client.yaml | 1 + .../thrash/clusters/fixed-3-cephfs.yaml | 1 - ceph/qa/suites/kcephfs/thrash/overrides/+ | 0 .../kcephfs/thrash/overrides/debug.yaml | 1 + .../kcephfs/thrash/overrides/frag_enable.yaml | 1 + .../kcephfs/thrash/overrides/log-config.yaml | 1 + .../kcephfs/thrash/overrides/osd-asserts.yaml | 1 + .../thrash/overrides/whitelist_health.yaml | 1 + .../whitelist_wrongly_marked_down.yaml | 1 + .../suites/kcephfs/thrash/thrashers/mon.yaml | 4 + .../rados/singleton/all/divergent_priors.yaml | 2 +- .../singleton/all/divergent_priors2.yaml | 2 +- .../suites/rbd/maintenance/qemu/xfstests.yaml | 1 + ceph/qa/suites/rgw/singleton/overrides.yaml | 2 - ceph/qa/suites/rgw/verify/overrides.yaml | 1 - .../jewel-x/ceph-deploy/jewel-luminous.yaml | 2 +- .../point-to-point-x/distros/centos_7.3.yaml | 1 - .../distros/ubuntu_14.04.yaml | 1 - .../point-to-point-upgrade.yaml | 236 -- .../point-to-point-upgrade.yaml | 8 +- ceph/qa/tasks/cephfs/filesystem.py | 7 +- ceph/qa/tasks/cephfs/test_full.py | 38 +- ceph/qa/tasks/cephfs/test_scrub.py | 157 + ceph/qa/tasks/mgr/test_module_selftest.py | 1 + ceph/qa/tasks/qemu.py | 7 + ceph/qa/tasks/swift.py | 5 +- .../workunits/rados/test-upgrade-to-mimic.sh | 52 + ceph/qa/workunits/rbd/run_devstack_tempest.sh | 2 + ceph/qa/workunits/rbd/test_admin_socket.sh | 2 - ceph/run-make-check.sh | 11 +- ceph/src/.git_version | 4 +- ceph/src/ceph-detect-init/CMakeLists.txt | 2 +- ceph/src/ceph-detect-init/tox.ini | 1 - ceph/src/ceph-disk/CMakeLists.txt | 2 +- ceph/src/ceph-disk/tox.ini | 3 +- ceph/src/ceph-volume/ceph_volume/api/lvm.py | 122 +- .../ceph_volume/devices/lvm/activate.py | 105 +- .../ceph_volume/devices/lvm/common.py | 7 + .../ceph_volume/devices/lvm/listing.py | 34 +- .../ceph_volume/devices/lvm/prepare.py | 13 +- ceph/src/ceph-volume/ceph_volume/process.py | 25 +- .../ceph_volume/systemd/systemctl.py | 14 +- .../ceph_volume/tests/api/test_lvm.py | 147 + .../tests/devices/lvm/test_activate.py | 188 + .../ceph_volume/tests/functional/Vagrantfile | 8 + .../lvm/centos7/bluestore/dmcrypt/test.yml | 35 +- .../lvm/centos7/filestore/dmcrypt/test.yml | 36 +- .../lvm/playbooks/test_bluestore.yml | 29 +- .../lvm/playbooks/test_filestore.yml | 36 +- .../ceph_volume/tests/functional/lvm/tox.ini | 5 +- .../lvm/xenial/bluestore/dmcrypt/test.yml | 23 + .../lvm/xenial/filestore/dmcrypt/test.yml | 36 +- .../functional/scripts/vagrant_reload.sh | 21 + .../tests/functional/simple/tox.ini | 3 +- .../ceph_volume/tests/test_process.py | 68 + .../ceph_volume/tests/util/test_prepare.py | 28 +- .../ceph_volume/tests/util/test_system.py | 26 + ceph/src/ceph-volume/ceph_volume/util/disk.py | 12 - .../ceph-volume/ceph_volume/util/prepare.py | 47 +- .../ceph-volume/ceph_volume/util/system.py | 25 +- ceph/src/client/Client.cc | 51 +- ceph/src/client/Inode.cc | 32 +- ceph/src/client/Inode.h | 2 +- ceph/src/client/MetaRequest.h | 2 +- ceph/src/client/SyntheticClient.cc | 2 +- ceph/src/cls/rbd/cls_rbd.cc | 2 +- ceph/src/cls/rgw/cls_rgw.cc | 26 +- ceph/src/cls/user/cls_user.cc | 49 +- ceph/src/cls/user/cls_user_client.cc | 9 + ceph/src/cls/user/cls_user_client.h | 1 + ceph/src/cls/user/cls_user_ops.h | 21 + ceph/src/common/Formatter.cc | 36 +- ceph/src/common/Formatter.h | 32 +- ceph/src/common/HTMLFormatter.cc | 8 +- ceph/src/common/HTMLFormatter.h | 4 +- ceph/src/common/ceph_argparse.cc | 30 +- ceph/src/common/ceph_time.h | 4 + ceph/src/common/dns_resolve.cc | 9 +- ceph/src/common/escape.c | 13 +- ceph/src/common/escape.h | 8 +- ceph/src/common/ipaddr.cc | 2 + ceph/src/common/legacy_config_opts.h | 5 +- ceph/src/common/options.cc | 25 +- ceph/src/common/scrub_types.cc | 24 +- ceph/src/common/scrub_types.h | 51 +- ceph/src/common/sstring.hh | 2 +- ceph/src/common/util.cc | 2 +- ceph/src/crush/CrushWrapper.cc | 21 +- ceph/src/crush/CrushWrapper.h | 7 + ceph/src/include/btree_interval_set.h | 585 ---- .../include/{encoding_btree.h => btree_map.h} | 7 +- ceph/src/include/compact_map.h | 77 +- ceph/src/include/compact_set.h | 67 +- ceph/src/include/config-h.in.cmake | 3 + ceph/src/include/encoding.h | 30 +- ceph/src/include/filepath.h | 35 +- ceph/src/include/frag.h | 6 +- ceph/src/include/interval_set.h | 335 +- ceph/src/include/mempool.h | 9 + ceph/src/include/rados/rados_types.hpp | 118 +- ceph/src/java/CMakeLists.txt | 2 +- ceph/src/journal/JournalMetadata.cc | 177 +- ceph/src/journal/JournalMetadata.h | 16 +- ceph/src/journal/ObjectRecorder.cc | 42 +- ceph/src/journal/ObjectRecorder.h | 1 + ceph/src/kv/MemDB.h | 4 +- ceph/src/librados/IoCtxImpl.cc | 2 +- ceph/src/librados/librados.cc | 5 +- ceph/src/librados/snap_set_diff.cc | 9 +- ceph/src/librados/snap_set_diff.h | 3 +- ceph/src/librbd/Journal.cc | 15 +- ceph/src/librbd/Operations.cc | 4 +- ceph/src/librbd/api/DiffIterate.cc | 11 +- ceph/src/librbd/io/ObjectRequest.h | 4 +- ceph/src/librbd/operation/ObjectMapIterate.cc | 8 +- ceph/src/librbd/operation/RenameRequest.cc | 4 +- ceph/src/librbd/operation/RenameRequest.h | 3 +- ceph/src/log/Entry.h | 13 + ceph/src/log/EntryQueue.h | 2 +- ceph/src/log/Log.cc | 2 +- ceph/src/mds/Beacon.cc | 2 +- ceph/src/mds/Beacon.h | 4 +- ceph/src/mds/CDentry.cc | 4 +- ceph/src/mds/CDentry.h | 46 +- ceph/src/mds/CDir.cc | 227 +- ceph/src/mds/CDir.h | 108 +- ceph/src/mds/CInode.cc | 580 ++- ceph/src/mds/CInode.h | 270 +- ceph/src/mds/Capability.cc | 28 +- ceph/src/mds/Capability.h | 15 +- ceph/src/mds/DamageTable.cc | 16 +- ceph/src/mds/DamageTable.h | 12 +- ceph/src/mds/FSMap.cc | 19 +- ceph/src/mds/FSMap.h | 25 +- ceph/src/mds/FSMapUser.h | 6 +- ceph/src/mds/Locker.cc | 141 +- ceph/src/mds/Locker.h | 12 +- ceph/src/mds/LogEvent.cc | 42 +- ceph/src/mds/LogEvent.h | 5 +- ceph/src/mds/MDBalancer.cc | 29 +- ceph/src/mds/MDCache.cc | 383 +- ceph/src/mds/MDCache.h | 18 +- ceph/src/mds/MDSAuthCaps.cc | 17 +- ceph/src/mds/MDSAuthCaps.h | 16 +- ceph/src/mds/MDSCacheObject.h | 24 +- ceph/src/mds/MDSDaemon.cc | 23 +- ceph/src/mds/MDSDaemon.h | 18 +- ceph/src/mds/MDSMap.h | 15 +- ceph/src/mds/MDSRank.cc | 32 +- ceph/src/mds/MDSRank.h | 28 +- ceph/src/mds/Mantle.cc | 4 +- ceph/src/mds/Mantle.h | 4 +- ceph/src/mds/Migrator.cc | 34 +- ceph/src/mds/RecoveryQueue.cc | 2 +- ceph/src/mds/ScrubHeader.h | 6 +- ceph/src/mds/ScrubStack.cc | 2 +- ceph/src/mds/Server.cc | 518 +-- ceph/src/mds/Server.h | 7 +- ceph/src/mds/SessionMap.cc | 4 +- ceph/src/mds/SimpleLock.cc | 5 +- ceph/src/mds/SimpleLock.h | 13 +- ceph/src/mds/SnapClient.h | 6 +- ceph/src/mds/SnapRealm.cc | 10 +- ceph/src/mds/SnapRealm.h | 8 +- ceph/src/mds/StrayManager.cc | 40 +- ceph/src/mds/events/EMetaBlob.h | 37 +- ceph/src/mds/inode_backtrace.h | 4 +- ceph/src/mds/journal.cc | 49 +- ceph/src/mds/locks.c | 18 +- ceph/src/mds/mdstypes.cc | 350 +- ceph/src/mds/mdstypes.h | 611 +++- ceph/src/mds/snap.cc | 4 +- ceph/src/mds/snap.h | 4 +- ceph/src/messages/MCacheExpire.h | 6 +- ceph/src/messages/MClientLease.h | 6 +- ceph/src/messages/MClientRequest.h | 4 +- ceph/src/messages/MCommandReply.h | 4 +- ceph/src/messages/MDentryLink.h | 4 +- ceph/src/messages/MDentryUnlink.h | 4 +- ceph/src/messages/MDirUpdate.h | 52 +- ceph/src/messages/MDiscoverReply.h | 10 +- ceph/src/messages/MMDSBeacon.h | 4 +- ceph/src/messages/MMDSCacheRejoin.h | 12 +- ceph/src/messages/MOSDPGUpdateLogMissing.h | 25 +- .../messages/MOSDPGUpdateLogMissingReply.h | 17 +- ceph/src/mgr/BaseMgrModule.cc | 14 + ceph/src/mgr/Mgr.cc | 6 +- ceph/src/mgr/MgrStandby.cc | 8 +- ceph/src/mgr/PyFormatter.cc | 4 +- ceph/src/mgr/PyFormatter.h | 2 +- ceph/src/mgr/PyOSDMap.cc | 23 +- ceph/src/mon/FSCommands.cc | 2 +- ceph/src/mon/MgrMonitor.cc | 1 + ceph/src/mon/MgrStatMonitor.cc | 1 + ceph/src/mon/MonClient.h | 2 + ceph/src/mon/OSDMonitor.cc | 159 +- ceph/src/mon/OSDMonitor.h | 3 +- ceph/src/mon/PGMap.cc | 6 +- ceph/src/mon/PaxosService.cc | 1 + ceph/src/msg/async/Event.cc | 2 +- ceph/src/ocf/CMakeLists.txt | 2 +- ceph/src/os/bluestore/BitMapAllocator.h | 1 - ceph/src/os/bluestore/BlueFS.cc | 78 +- ceph/src/os/bluestore/BlueFS.h | 2 +- ceph/src/os/bluestore/BlueStore.cc | 77 +- ceph/src/os/bluestore/BlueStore.h | 3 +- ceph/src/os/bluestore/KernelDevice.cc | 2 +- ceph/src/os/bluestore/StupidAllocator.cc | 27 +- ceph/src/os/bluestore/StupidAllocator.h | 11 +- ceph/src/os/bluestore/aio.h | 4 +- ceph/src/os/bluestore/bluefs_types.cc | 1 + ceph/src/os/bluestore/bluefs_types.h | 44 +- ceph/src/os/bluestore/bluestore_types.h | 8 +- ceph/src/os/filestore/FileStore.cc | 2 +- ceph/src/osd/ECBackend.cc | 7 +- ceph/src/osd/ECUtil.cc | 12 +- ceph/src/osd/ECUtil.h | 2 + ceph/src/osd/OSD.cc | 91 +- ceph/src/osd/OSD.h | 24 +- ceph/src/osd/OSDMap.cc | 135 + ceph/src/osd/OSDMap.h | 16 +- ceph/src/osd/PG.cc | 95 +- ceph/src/osd/PG.h | 10 +- ceph/src/osd/PGBackend.cc | 63 +- ceph/src/osd/PGLog.h | 2 +- ceph/src/osd/PrimaryLogPG.cc | 162 +- ceph/src/osd/PrimaryLogPG.h | 17 +- ceph/src/osd/SnapMapper.cc | 8 +- ceph/src/osd/mClockClientQueue.cc | 4 +- ceph/src/osd/mClockOpClassQueue.h | 4 +- ceph/src/osd/osd_types.cc | 27 +- ceph/src/osd/osd_types.h | 43 +- ceph/src/osdc/Journaler.cc | 11 +- ceph/src/osdc/Journaler.h | 6 +- ceph/src/osdc/Objecter.cc | 12 +- ceph/src/osdc/Objecter.h | 1 + ceph/src/pybind/ceph_argparse.py | 35 +- ceph/src/pybind/mgr/balancer/module.py | 325 +- ceph/src/pybind/mgr/localpool/module.py | 1 - ceph/src/pybind/mgr/mgr_module.py | 27 +- ceph/src/pybind/mgr/prometheus/module.py | 557 ++- ceph/src/pybind/mgr/zabbix/module.py | 62 +- ceph/src/rgw/CMakeLists.txt | 5 + ceph/src/rgw/librgw.cc | 5 + ceph/src/rgw/rgw_admin.cc | 123 +- ceph/src/rgw/rgw_asio_frontend.cc | 34 +- ceph/src/rgw/rgw_asio_frontend.h | 2 +- ceph/src/rgw/rgw_auth_s3.cc | 70 +- ceph/src/rgw/rgw_auth_s3.h | 5 + ceph/src/rgw/rgw_bucket.cc | 30 + ceph/src/rgw/rgw_bucket.h | 8 + ceph/src/rgw/rgw_civetweb.cc | 19 +- ceph/src/rgw/rgw_civetweb.h | 1 + ceph/src/rgw/rgw_civetweb_frontend.cc | 11 +- ceph/src/rgw/rgw_common.cc | 6 +- ceph/src/rgw/rgw_common.h | 3 + ceph/src/rgw/rgw_compression.cc | 18 +- ceph/src/rgw/rgw_data_sync.cc | 7 +- ceph/src/rgw/rgw_data_sync.h | 6 + ceph/src/rgw/rgw_file.cc | 2 +- ceph/src/rgw/rgw_file.h | 26 +- ceph/src/rgw/rgw_formats.cc | 4 +- ceph/src/rgw/rgw_formats.h | 2 +- ceph/src/rgw/rgw_http_client.cc | 134 +- ceph/src/rgw/rgw_http_client_curl.cc | 122 + ceph/src/rgw/rgw_http_client_curl.h | 30 + ceph/src/rgw/rgw_log.cc | 32 +- ceph/src/rgw/rgw_main.cc | 31 +- ceph/src/rgw/rgw_op.cc | 3 + ceph/src/rgw/rgw_process.cc | 11 +- ceph/src/rgw/rgw_process.h | 3 +- ceph/src/rgw/rgw_rados.cc | 38 +- ceph/src/rgw/rgw_rados.h | 1 + ceph/src/rgw/rgw_reshard.cc | 18 +- ceph/src/rgw/rgw_reshard.h | 1 + ceph/src/rgw/rgw_rest.cc | 2 +- ceph/src/rgw/rgw_rest_bucket.cc | 80 + ceph/src/rgw/rgw_rest_s3.cc | 72 +- ceph/src/rgw/rgw_rest_s3.h | 5 - ceph/src/rgw/rgw_rest_s3website.h | 3 + ceph/src/rgw/rgw_rest_user.cc | 5 +- ceph/src/rocksdb/CMakeLists.txt | 18 + ceph/src/rocksdb/util/crc32c.cc | 9 - .../src/test/cli/ceph-authtool/add-key-segv.t | 2 +- ceph/src/test/cli/ceph-authtool/add-key.t | 2 +- ceph/src/test/cli/ceph-authtool/cap-bin.t | 2 +- ceph/src/test/cli/ceph-authtool/cap-invalid.t | 2 +- .../test/cli/ceph-authtool/cap-overwrite.t | 2 +- ceph/src/test/cli/ceph-authtool/cap.t | 2 +- .../cli/ceph-authtool/create-gen-list-bin.t | 3 +- .../test/cli/ceph-authtool/create-gen-list.t | 2 +- ceph/src/test/cli/ceph-authtool/help.t | 2 + ceph/src/test/cli/ceph-authtool/manpage.t | 2 + ceph/src/test/cli/ceph-authtool/simple.t | 2 + ceph/src/test/cli/crushtool/add-bucket.t | 69 + ceph/src/test/cli/crushtool/help.t | 5 + ceph/src/test/cli/radosgw-admin/help.t | 5 +- ceph/src/test/common/test_interval_set.cc | 91 +- ceph/src/test/common/test_util.cc | 1 - ceph/src/test/encoding/check-generated.sh | 21 +- ceph/src/test/encoding/types.h | 4 +- ceph/src/test/fs/mds_types.cc | 12 +- ceph/src/test/libcephfs/deleg.cc | 106 +- ceph/src/test/libcephfs/test.cc | 5 +- .../librados_test_stub/MockTestMemIoCtxImpl.h | 6 + .../test/librbd/io/test_mock_ObjectRequest.cc | 13 +- ceph/src/test/librbd/test_librbd.cc | 9 +- ceph/src/test/librbd/test_mock_Journal.cc | 13 + ceph/src/test/objectstore/store_test.cc | 106 +- ceph/src/test/osd/TestOSDMap.cc | 291 +- ceph/src/test/osd/TestPGLog.cc | 21 - ceph/src/test/pybind/test_ceph_argparse.py | 40 +- ceph/src/test/pybind/test_rados.py | 4 +- ceph/src/test/pybind/test_rbd.py | 2 + ceph/src/tools/CMakeLists.txt | 3 +- ceph/src/tools/ceph_authtool.cc | 14 +- ceph/src/tools/ceph_objectstore_tool.cc | 127 +- ceph/src/tools/cephfs/DataScan.cc | 2 +- ceph/src/tools/cephfs/JournalTool.cc | 4 +- ceph/src/tools/cephfs/MDSUtility.cc | 7 +- ceph/src/tools/cephfs/MDSUtility.h | 1 + ceph/src/tools/cephfs/cephfs-data-scan.cc | 1 - ceph/src/tools/cephfs/cephfs-journal-tool.cc | 2 - ceph/src/tools/cephfs/cephfs-table-tool.cc | 2 - ceph/src/tools/crushtool.cc | 124 +- ceph/src/tools/rados/rados.cc | 151 +- .../image_replayer/ReplayStatusFormatter.cc | 28 +- .../image_sync/ObjectCopyRequest.cc | 19 +- .../rbd_mirror/image_sync/ObjectCopyRequest.h | 1 + ceph/src/tools/rbd_nbd/rbd-nbd.cc | 70 +- ceph/src/tools/setup-virtualenv.sh | 4 +- ceph/systemd/ceph-mgr@.service | 1 + 410 files changed, 12269 insertions(+), 5072 deletions(-) create mode 100644 ceph/doc/rados/operations/bluestore-migration.rst create mode 100644 ceph/doc/radosgw/frontends.rst create mode 100644 ceph/qa/cephfs/clusters/1-mds-1-client.yaml create mode 100644 ceph/qa/cephfs/clusters/1-mds-2-client.yaml create mode 100644 ceph/qa/cephfs/clusters/1-mds-4-client.yaml create mode 100644 ceph/qa/cephfs/overrides/log-config.yaml create mode 100644 ceph/qa/cephfs/overrides/osd-asserts.yaml create mode 100755 ceph/qa/standalone/osd/repro_long_log.sh create mode 100755 ceph/qa/standalone/scrub/osd-scrub-test.sh create mode 120000 ceph/qa/suites/kcephfs/cephfs/clusters/1-mds-1-client.yaml delete mode 120000 ceph/qa/suites/kcephfs/cephfs/clusters/fixed-3-cephfs.yaml rename ceph/qa/suites/{upgrade/jewel-x/point-to-point-x/% => kcephfs/cephfs/overrides/+} (100%) create mode 120000 ceph/qa/suites/kcephfs/cephfs/overrides/debug.yaml create mode 120000 ceph/qa/suites/kcephfs/cephfs/overrides/frag_enable.yaml create mode 120000 ceph/qa/suites/kcephfs/cephfs/overrides/log-config.yaml create mode 120000 ceph/qa/suites/kcephfs/cephfs/overrides/osd-asserts.yaml create mode 120000 ceph/qa/suites/kcephfs/cephfs/overrides/whitelist_health.yaml create mode 120000 ceph/qa/suites/kcephfs/cephfs/overrides/whitelist_wrongly_marked_down.yaml create mode 120000 ceph/qa/suites/kcephfs/mixed-clients/clusters/1-mds-2-client.yaml delete mode 100644 ceph/qa/suites/kcephfs/mixed-clients/clusters/2-clients.yaml create mode 100644 ceph/qa/suites/kcephfs/mixed-clients/overrides/+ create mode 120000 ceph/qa/suites/kcephfs/mixed-clients/overrides/debug.yaml create mode 120000 ceph/qa/suites/kcephfs/mixed-clients/overrides/frag_enable.yaml create mode 120000 ceph/qa/suites/kcephfs/mixed-clients/overrides/log-config.yaml create mode 120000 ceph/qa/suites/kcephfs/mixed-clients/overrides/osd-asserts.yaml create mode 120000 ceph/qa/suites/kcephfs/mixed-clients/overrides/whitelist_health.yaml create mode 120000 ceph/qa/suites/kcephfs/mixed-clients/overrides/whitelist_wrongly_marked_down.yaml create mode 120000 ceph/qa/suites/kcephfs/recovery/clusters/1-mds-4-client.yaml delete mode 100644 ceph/qa/suites/kcephfs/recovery/clusters/4-remote-clients.yaml create mode 100644 ceph/qa/suites/kcephfs/recovery/overrides/+ create mode 120000 ceph/qa/suites/kcephfs/recovery/overrides/debug.yaml create mode 120000 ceph/qa/suites/kcephfs/recovery/overrides/frag_enable.yaml create mode 120000 ceph/qa/suites/kcephfs/recovery/overrides/log-config.yaml create mode 120000 ceph/qa/suites/kcephfs/recovery/overrides/osd-asserts.yaml create mode 120000 ceph/qa/suites/kcephfs/recovery/overrides/whitelist_health.yaml create mode 120000 ceph/qa/suites/kcephfs/recovery/overrides/whitelist_wrongly_marked_down.yaml create mode 120000 ceph/qa/suites/kcephfs/thrash/clusters/1-mds-1-client.yaml delete mode 120000 ceph/qa/suites/kcephfs/thrash/clusters/fixed-3-cephfs.yaml create mode 100644 ceph/qa/suites/kcephfs/thrash/overrides/+ create mode 120000 ceph/qa/suites/kcephfs/thrash/overrides/debug.yaml create mode 120000 ceph/qa/suites/kcephfs/thrash/overrides/frag_enable.yaml create mode 120000 ceph/qa/suites/kcephfs/thrash/overrides/log-config.yaml create mode 120000 ceph/qa/suites/kcephfs/thrash/overrides/osd-asserts.yaml create mode 120000 ceph/qa/suites/kcephfs/thrash/overrides/whitelist_health.yaml create mode 120000 ceph/qa/suites/kcephfs/thrash/overrides/whitelist_wrongly_marked_down.yaml delete mode 120000 ceph/qa/suites/upgrade/jewel-x/point-to-point-x/distros/centos_7.3.yaml delete mode 120000 ceph/qa/suites/upgrade/jewel-x/point-to-point-x/distros/ubuntu_14.04.yaml delete mode 100644 ceph/qa/suites/upgrade/jewel-x/point-to-point-x/point-to-point-upgrade.yaml create mode 100644 ceph/qa/tasks/cephfs/test_scrub.py create mode 100755 ceph/qa/workunits/rados/test-upgrade-to-mimic.sh create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/scripts/vagrant_reload.sh create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/test_process.py delete mode 100644 ceph/src/include/btree_interval_set.h rename ceph/src/include/{encoding_btree.h => btree_map.h} (87%) create mode 100644 ceph/src/rgw/rgw_http_client_curl.cc create mode 100644 ceph/src/rgw/rgw_http_client_curl.h create mode 100644 ceph/src/test/cli/crushtool/add-bucket.t diff --git a/Makefile b/Makefile index 6a15306fa..3c2451983 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ RELEASE=5.1 PACKAGE=ceph -VER=12.2.4 +VER=12.2.5 DEBREL=pve1 SRCDIR=ceph diff --git a/ceph/CMakeLists.txt b/ceph/CMakeLists.txt index 503305ede..aa90ba65d 100644 --- a/ceph/CMakeLists.txt +++ b/ceph/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 2.8.11) project(ceph) -set(VERSION 12.2.4) +set(VERSION 12.2.5) if(POLICY CMP0046) # Tweak policies (this one disables "missing" dependency warning) @@ -389,7 +389,24 @@ if (WITH_RADOSGW) message(STATUS "Looking for openssl anyways, because radosgw selected") find_package(OpenSSL) endif() +# https://curl.haxx.se/docs/install.html mentions the +# configure flags for various ssl backends + execute_process( + COMMAND + "sh" "-c" + "curl-config --configure | grep with-ssl" + RESULT_VARIABLE NO_CURL_SSL_LINK + ERROR_VARIABLE CURL_CONFIG_ERRORS + ) + if (CURL_CONFIG_ERRORS) + message(WARNING "unable to run curl-config; rgw cannot make ssl requests to external systems reliably") + endif() + find_package(OpenSSL) if (OPENSSL_FOUND) + if (NOT NO_CURL_SSL_LINK) + message(STATUS "libcurl is linked with openssl: explicitly setting locks") + set(WITH_CURL_OPENSSL ON) + endif() # CURL_SSL_LINK execute_process( COMMAND "sh" "-c" @@ -422,7 +439,7 @@ if (WITH_RADOSGW) message(STATUS "crypto soname: ${LIBCRYPTO_SONAME}") else() message(WARNING "ssl not found: rgw civetweb may fail to dlopen libssl libcrypto") - endif() + endif() # OPENSSL_FOUND endif (WITH_RADOSGW) #option for CephFS diff --git a/ceph/COPYING b/ceph/COPYING index a87427936..cd45ce086 100644 --- a/ceph/COPYING +++ b/ceph/COPYING @@ -9,7 +9,7 @@ License: LGPL2.1 (see COPYING-LGPL2.1) Files: doc/* Copyright: (c) 2010-2012 New Dream Network and contributors -License: Creative Commons Attribution-ShareAlike (CC BY-SA) +License: Creative Commons Attribution Share Alike 3.0 (CC-BY-SA-3.0) Files: bin/git-archive-all.sh License: GPL3 diff --git a/ceph/PendingReleaseNotes b/ceph/PendingReleaseNotes index ac6b2c0c6..139ca0fc7 100644 --- a/ceph/PendingReleaseNotes +++ b/ceph/PendingReleaseNotes @@ -54,3 +54,60 @@ and refuses to start as consistency and untrimmable cache issues may develop. The new option client_die_on_failed_dentry_invalidate (default: true) may be turned off to allow the client to proceed (dangerous!). + +12.2.5 +------ + +- *CephFS*: + + * Upgrading an MDS cluster to 12.2.3+ will result in all active MDS + exiting due to feature incompatibilities once an upgraded MDS comes online + (even as standby). Operators may ignore the error messages and continue + upgrading/restarting or follow this upgrade sequence: + + Reduce the number of ranks to 1 (`ceph fs set max_mds 1`), + deactivate all other ranks (`ceph mds deactivate :`), shutdown + standbys leaving the one active MDS, upgrade the single active MDS, then + upgrade/start standbys. Finally, restore the previous max_mds. + + See also: https://tracker.ceph.com/issues/23172 + +* *rados list-inconsistent-obj format changes:* + + * Various error strings have been improved. For example, the "oi" or "oi_attr" + in errors which stands for object info is now "info" (e.g. oi_attr_missing is + now info_missing). + + * The object's "selected_object_info" is now in json format instead of string. + + * The attribute errors (attr_value_mismatch, attr_name_mismatch) only apply to user + attributes. Only user attributes are output and have the internal leading underscore + stripped. + + * If there are hash information errors (hinfo_missing, hinfo_corrupted, + hinfo_inconsistency) then "hashinfo" is added with the json format of the + information. If the information is corrupt then "hashinfo" is a string + containing the value. + + * If there are snapset errors (snapset_missing, snapset_corrupted, + snapset_inconsistency) then "snapset" is added with the json format of the + information. If the information is corrupt then "snapset" is a string containing + the value. + + * If there are object information errors (info_missing, info_corrupted, + obj_size_info_mismatch, object_info_inconsistency) then "object_info" is added + with the json format of the information instead of a string. If the information + is corrupt then "object_info" is a string containing the value. + +* *rados list-inconsistent-snapset format changes:* + + * Various error strings have been improved. For example, the "ss_attr" in + errors which stands for snapset info is now "snapset" (e.g. ss_attr_missing is + now snapset_missing). The error snapset_mismatch has been renamed to snapset_error + to better reflect what it means. + + * The head snapset information is output in json format as "snapset." This means that + even when there are no head errors, the head object will be output when any shard + has an error. This head object is there to show the snapset that was used in + determining errors. + diff --git a/ceph/README.md b/ceph/README.md index 93dc62bc2..c5bcf4e26 100644 --- a/ceph/README.md +++ b/ceph/README.md @@ -8,7 +8,7 @@ Please see http://ceph.com/ for current info. Most of Ceph is licensed under the LGPL version 2.1. Some miscellaneous code is under BSD-style license or is public domain. The documentation is licensed under Creative Commons -Attribution-ShareAlike (CC BY-SA). There are a handful of headers +Attribution Share Alike 3.0 (CC-BY-SA-3.0). There are a handful of headers included here that are licensed under the GPL. Please see the file COPYING for a full inventory of licenses by file. diff --git a/ceph/alpine/APKBUILD b/ceph/alpine/APKBUILD index db7465378..d280aacfb 100644 --- a/ceph/alpine/APKBUILD +++ b/ceph/alpine/APKBUILD @@ -1,7 +1,7 @@ # Contributor: John Coyle # Maintainer: John Coyle pkgname=ceph -pkgver=12.2.4 +pkgver=12.2.5 pkgrel=0 pkgdesc="Ceph is a distributed object store and file system" pkgusers="ceph" @@ -63,7 +63,7 @@ makedepends=" xmlstarlet yasm " -source="ceph-12.2.4.tar.bz2" +source="ceph-12.2.5.tar.bz2" subpackages=" $pkgname-base $pkgname-common @@ -116,7 +116,7 @@ _sysconfdir=/etc _udevrulesdir=/etc/udev/rules.d _python_sitelib=/usr/lib/python2.7/site-packages -builddir=$srcdir/ceph-12.2.4 +builddir=$srcdir/ceph-12.2.5 build() { export CEPH_BUILD_VIRTUALENV=$builddir diff --git a/ceph/ceph.spec b/ceph/ceph.spec index 38dab19eb..bbb86d452 100644 --- a/ceph/ceph.spec +++ b/ceph/ceph.spec @@ -61,7 +61,7 @@ # main package definition ################################################################################# Name: ceph -Version: 12.2.4 +Version: 12.2.5 Release: 0%{?dist} %if 0%{?fedora} || 0%{?rhel} Epoch: 2 @@ -72,12 +72,12 @@ Epoch: 2 %global _epoch_prefix %{?epoch:%{epoch}:} Summary: User space components of the Ceph file system -License: LGPL-2.1 and CC-BY-SA-1.0 and GPL-2.0 and BSL-1.0 and BSD-3-Clause and MIT +License: LGPL-2.1 and CC-BY-SA-3.0 and GPL-2.0 and BSL-1.0 and BSD-3-Clause and MIT %if 0%{?suse_version} Group: System/Filesystems %endif URL: http://ceph.com/ -Source0: http://ceph.com/download/ceph-12.2.4.tar.bz2 +Source0: http://ceph.com/download/ceph-12.2.5.tar.bz2 %if 0%{?suse_version} %if 0%{?is_opensuse} ExclusiveArch: x86_64 aarch64 ppc64 ppc64le @@ -112,6 +112,7 @@ BuildRequires: python-CherryPy BuildRequires: python-Werkzeug BuildRequires: python-numpy-devel %endif +BuildRequires: python-coverage BuildRequires: python-pecan BuildRequires: socat %endif @@ -778,7 +779,7 @@ python-rbd, python-rgw or python-cephfs instead. # common ################################################################################# %prep -%autosetup -p1 -n ceph-12.2.4 +%autosetup -p1 -n ceph-12.2.5 %build %if 0%{with cephfs_java} diff --git a/ceph/ceph.spec.in b/ceph/ceph.spec.in index ad4ef62e5..2f89a9bc1 100644 --- a/ceph/ceph.spec.in +++ b/ceph/ceph.spec.in @@ -72,7 +72,7 @@ Epoch: 2 %global _epoch_prefix %{?epoch:%{epoch}:} Summary: User space components of the Ceph file system -License: LGPL-2.1 and CC-BY-SA-1.0 and GPL-2.0 and BSL-1.0 and BSD-3-Clause and MIT +License: LGPL-2.1 and CC-BY-SA-3.0 and GPL-2.0 and BSL-1.0 and BSD-3-Clause and MIT %if 0%{?suse_version} Group: System/Filesystems %endif @@ -112,6 +112,7 @@ BuildRequires: python-CherryPy BuildRequires: python-Werkzeug BuildRequires: python-numpy-devel %endif +BuildRequires: python-coverage BuildRequires: python-pecan BuildRequires: socat %endif diff --git a/ceph/debian/changelog b/ceph/debian/changelog index d8c329033..c3a8efa2d 100644 --- a/ceph/debian/changelog +++ b/ceph/debian/changelog @@ -1,3 +1,9 @@ +ceph (12.2.5-1) stable; urgency=medium + + * New upstream release + + -- Ceph Release Team Mon, 23 Apr 2018 16:18:32 +0000 + ceph (12.2.4-1) stable; urgency=medium * New upstream release diff --git a/ceph/debian/control b/ceph/debian/control index 9940e7491..bdc74471d 100644 --- a/ceph/debian/control +++ b/ceph/debian/control @@ -29,7 +29,7 @@ Build-Depends: bc, libbabeltrace-ctf-dev, libbabeltrace-dev, libblkid-dev (>= 2.17), - libcurl4-gnutls-dev, + libcurl4-openssl-dev, libexpat1-dev, libfuse-dev, libgoogle-perftools-dev [i386 amd64 arm64], @@ -49,8 +49,8 @@ Build-Depends: bc, pkg-config, python (>= 2.7), python-all-dev, + python-coverage, python-cherrypy3, - python-jinja2, python-nose, python-pecan, python-prettytable, @@ -169,6 +169,7 @@ Package: ceph-mgr Architecture: linux-any Depends: ceph-base (= ${binary:Version}), python-cherrypy3, + python-jinja2, python-openssl, python-pecan, python-werkzeug, diff --git a/ceph/debian/copyright b/ceph/debian/copyright index 994d67607..8b0fcd27b 100644 --- a/ceph/debian/copyright +++ b/ceph/debian/copyright @@ -15,7 +15,7 @@ License: BSD 3-clause Files: doc/* Copyright: (c) 2010-2012 New Dream Network and contributors -License: Creative Commons Attribution-ShareAlike (CC BY-SA) +License: Creative Commons Attribution Share Alike 3.0 (CC-BY-SA-3.0) Files: src/mount/canonicalize.c Copyright: Copyright (C) 1993 Rick Sladkey diff --git a/ceph/doc/cephfs/upgrading.rst b/ceph/doc/cephfs/upgrading.rst index 7ee3f0986..c29644b60 100644 --- a/ceph/doc/cephfs/upgrading.rst +++ b/ceph/doc/cephfs/upgrading.rst @@ -1,3 +1,51 @@ +Upgrading the MDS Cluster +========================= + +Currently the MDS cluster does not have built-in versioning or file system +flags to support seamless upgrades of the MDSs without potentially causing +assertions or other faults due to incompatible messages or other functional +differences. For this reason, it's necessary during any cluster upgrade to +reduce the number of active MDS for a file system to one first so that two +active MDS do not communicate with different versions. Further, it's also +necessary to take standbys offline as any new CompatSet flags will propogate +via the MDSMap to all MDS and cause older MDS to suicide. + +The proper sequence for upgrading the MDS cluster is: + +1. Reduce the number of ranks to 1: + +:: + + ceph fs set max_mds 1 + +2. Deactivate all non-zero ranks, from the highest rank to the lowest, while waiting for each MDS to finish stopping: + +:: + + ceph mds deactivate : + ceph status # wait for MDS to finish stopping + +3. Take all standbys offline, e.g. using systemctl: + +:: + + systemctl stop ceph-mds.target + ceph status # confirm only one MDS is online and is active + +4. Upgrade the single active MDS, e.g. using systemctl: + +:: + + systemctl restart ceph-mds.target + +5. Upgrade/start the standby daemons. + +6. Restore the previous max_mds for your cluster: + +:: + + ceph fs set max_mds + Upgrading pre-Firefly filesystems past Jewel ============================================ diff --git a/ceph/doc/conf.py b/ceph/doc/conf.py index a1968bb4c..ce1e5af97 100644 --- a/ceph/doc/conf.py +++ b/ceph/doc/conf.py @@ -2,7 +2,7 @@ import sys import os project = u'Ceph' -copyright = u'2016, Red Hat, Inc, and contributors. Licensed under Creative Commons BY-SA' +copyright = u'2016, Red Hat, Inc, and contributors. Licensed under Creative Commons Attribution Share Alike 3.0 (CC-BY-SA-3.0)' version = 'dev' release = 'dev' diff --git a/ceph/doc/dev/ceph-volume/lvm.rst b/ceph/doc/dev/ceph-volume/lvm.rst index ceac235cd..b3e6bee12 100644 --- a/ceph/doc/dev/ceph-volume/lvm.rst +++ b/ceph/doc/dev/ceph-volume/lvm.rst @@ -166,3 +166,14 @@ the partition UUID. Example:: ceph.wal_uuid=A58D1C68-0D6E-4CB3-8E99-B261AD47CC39 + + +``vdo`` +------- +A VDO-enabled device is detected when device is getting prepared, and then +stored for later checks when activating. This affects mount options by +appending the ``discard`` mount flag, regardless of mount flags being used. + +Example for an enabled VDO device:: + + ceph.vdo=1 diff --git a/ceph/doc/dev/index.rst b/ceph/doc/dev/index.rst index b76f2f2fb..c22cd6d98 100644 --- a/ceph/doc/dev/index.rst +++ b/ceph/doc/dev/index.rst @@ -4,7 +4,7 @@ Contributing to Ceph: A Guide for Developers :Author: Loic Dachary :Author: Nathan Cutler -:License: Creative Commons Attribution-ShareAlike (CC BY-SA) +:License: Creative Commons Attribution Share Alike 3.0 (CC-BY-SA-3.0) .. note:: The old (pre-2016) developer documentation has been moved to :doc:`/dev/index-old`. diff --git a/ceph/doc/install/install-ceph-gateway.rst b/ceph/doc/install/install-ceph-gateway.rst index 8c4d96d3b..dc1178e5a 100644 --- a/ceph/doc/install/install-ceph-gateway.rst +++ b/ceph/doc/install/install-ceph-gateway.rst @@ -159,13 +159,21 @@ CA or intermediate certificates be supplied in one file. Each of these items must be in `pem` form. Because the combined file contains the secret key, it should be protected from unauthorized access. -To configure ssl operation, append ``s`` to the port number. Currently -it is not possible to configure the radosgw to listen on both -http and https, you must pick only one. So:: +To configure ssl operation, append ``s`` to the port number. For eg:: [client.rgw.gateway-node1] rgw_frontends = civetweb port=443s ssl_certificate=/etc/ceph/private/keyandcert.pem +.. versionadded :: Luminous + +Furthermore, civetweb can be made to bind to multiple ports, by separating them +with ``+`` in the configuration. This allows for use cases where both ssl and +non-ssl connections are hosted by a single rgw instance. For eg:: + + [client.rgw.gateway-node1] + rgw_frontends = civetweb port=80+443s ssl_certificate=/etc/ceph/private/keyandcert.pem + + Migrating from Apache to Civetweb --------------------------------- diff --git a/ceph/doc/install/manual-deployment.rst b/ceph/doc/install/manual-deployment.rst index affab5be9..8899c92fb 100644 --- a/ceph/doc/install/manual-deployment.rst +++ b/ceph/doc/install/manual-deployment.rst @@ -230,8 +230,8 @@ The procedure is as follows: auth service required = cephx auth client required = cephx osd journal size = 1024 - osd pool default size = 2 - osd pool default min size = 1 + osd pool default size = 3 + osd pool default min size = 2 osd pool default pg num = 333 osd pool default pgp num = 333 osd crush chooseleaf type = 1 @@ -311,36 +311,90 @@ a Ceph Node. Short Form ---------- -Ceph provides the ``ceph-disk`` utility, which can prepare a disk, partition or -directory for use with Ceph. The ``ceph-disk`` utility creates the OSD ID by -incrementing the index. Additionally, ``ceph-disk`` will add the new OSD to the -CRUSH map under the host for you. Execute ``ceph-disk -h`` for CLI details. -The ``ceph-disk`` utility automates the steps of the `Long Form`_ below. To +Ceph provides the ``ceph-volume`` utility, which can prepare a logical volume, disk, or partition +for use with Ceph. The ``ceph-volume`` utility creates the OSD ID by +incrementing the index. Additionally, ``ceph-volume`` will add the new OSD to the +CRUSH map under the host for you. Execute ``ceph-volume -h`` for CLI details. +The ``ceph-volume`` utility automates the steps of the `Long Form`_ below. To create the first two OSDs with the short form procedure, execute the following on ``node2`` and ``node3``: +bluestore +^^^^^^^^^ +#. Create the OSD. :: + + ssh {node-name} + sudo ceph-volume lvm create --data {data-path} + + For example:: + + ssh node1 + sudo ceph-volume lvm create --data /dev/hdd1 + +Alternatively, the creation process can be split in two phases (prepare, and +activate): #. Prepare the OSD. :: ssh {node-name} - sudo ceph-disk prepare --cluster {cluster-name} --cluster-uuid {uuid} {data-path} [{journal-path}] + sudo ceph-volume lvm prepare --data {data-path} {data-path} For example:: ssh node1 - sudo ceph-disk prepare --cluster ceph --cluster-uuid a7f64266-0894-4f1e-a635-d0aeaca0e993 --fs-type ext4 /dev/hdd1 + sudo ceph-volume lvm prepare --data /dev/hdd1 + + Once prepared, the ``ID`` and ``FSID`` of the prepared OSD are required for + activation. These can be obtained by listing OSDs in the current server:: + sudo ceph-volume lvm list #. Activate the OSD:: - sudo ceph-disk activate {data-path} [--activate-key {path}] + sudo ceph-volume lvm activate {ID} {FSID} + + For example:: + + sudo ceph-volume lvm activate 0 a7f64266-0894-4f1e-a635-d0aeaca0e993 + + +filestore +^^^^^^^^^ +#. Create the OSD. :: + + ssh {node-name} + sudo ceph-volume lvm create --filestore --data {data-path} --journal {journal-path} + + For example:: + + ssh node1 + sudo ceph-volume lvm create --filestore --data /dev/hdd1 --journal /dev/hdd2 + +Alternatively, the creation process can be split in two phases (prepare, and +activate): + +#. Prepare the OSD. :: + + ssh {node-name} + sudo ceph-volume lvm prepare --filestore --data {data-path} --journal {journal-path} For example:: - sudo ceph-disk activate /dev/hdd1 + ssh node1 + sudo ceph-volume lvm prepare --filestore --data /dev/hdd1 --journal /dev/hdd2 + + Once prepared, the ``ID`` and ``FSID`` of the prepared OSD are required for + activation. These can be obtained by listing OSDs in the current server:: + + sudo ceph-volume lvm list + +#. Activate the OSD:: + + sudo ceph-volume lvm activate --filestore {ID} {FSID} + + For example:: - **Note:** Use the ``--activate-key`` argument if you do not have a copy - of ``/var/lib/ceph/bootstrap-osd/{cluster}.keyring`` on the Ceph Node. + sudo ceph-volume lvm activate --filestore 0 a7f64266-0894-4f1e-a635-d0aeaca0e993 Long Form diff --git a/ceph/doc/man/8/ceph-authtool.rst b/ceph/doc/man/8/ceph-authtool.rst index f1ac1521e..fb0afe7da 100644 --- a/ceph/doc/man/8/ceph-authtool.rst +++ b/ceph/doc/man/8/ceph-authtool.rst @@ -21,6 +21,7 @@ Synopsis [ -a | --add-key *base64_key* ] [ --cap *subsystem* *capability* ] [ --caps *capfile* ] + [ --mode *mode* ] Description @@ -87,6 +88,10 @@ Options will set all of capabilities associated with a given key, for all subsystems + .. option:: --mode *mode* + + will set the desired file mode to the keyring e.g: 0644, defaults to 0600 + Capabilities ============ @@ -172,9 +177,9 @@ value is the capability string (see above). Example ======= -To create a new keyring containing a key for client.foo:: +To create a new keyring containing a key for client.foo with a 0644 file mode:: - ceph-authtool -C -n client.foo --gen-key keyring + ceph-authtool -C -n client.foo --gen-key keyring --mode 0644 To associate some capabilities with the key (namely, the ability to mount a Ceph filesystem):: diff --git a/ceph/doc/man/8/ceph-volume.rst b/ceph/doc/man/8/ceph-volume.rst index f677d0623..d38cb624e 100644 --- a/ceph/doc/man/8/ceph-volume.rst +++ b/ceph/doc/man/8/ceph-volume.rst @@ -50,18 +50,23 @@ enabled and needs to be mounted. Usage:: - ceph-volume lvm activate --filestore + ceph-volume lvm activate --bluestore Optional Arguments: * [-h, --help] show the help message and exit -* [--bluestore] filestore objectstore (not yet implemented) -* [--filestore] filestore objectstore (current default) +* [--auto-detect-objectstore] Automatically detect the objecstore by inspecting + the OSD +* [--bluestore] bluestore objectstore (default) +* [--filestore] filestore objectstore +* [--all] Activate all OSDs found in the system +* [--no-systemd] Skip creating and enabling systemd units and starting of OSD + services **prepare** -Prepares a logical volume to be used as an OSD and journal using a ``filestore`` setup -(``bluestore`` support is planned). It will not create or modify the logical volumes +Prepares a logical volume to be used as an OSD and journal using a ``filestore`` +or ``bluestore`` (default) setup. It will not create or modify the logical volumes except for adding extra metadata. Usage:: @@ -72,12 +77,14 @@ Optional arguments: * [-h, --help] show the help message and exit * [--journal JOURNAL] A logical group name, path to a logical volume, or path to a device -* [--journal-size GB] Size (in GB) A logical group name or a path to a logical volume * [--bluestore] Use the bluestore objectstore (default) +* [--block.wal] Path to a bluestore block.wal logical volume or partition +* [--block.db] Path to a bluestore block.db logical volume or partition * [--filestore] Use the filestore objectstore * [--dmcrypt] Enable encryption for the underlying OSD devices * [--osd-id OSD_ID] Reuse an existing OSD id * [--osd-fsid OSD_FSID] Reuse an existing OSD fsid +* [--crush-device-class] Define a CRUSH device class to assign the OSD to Required arguments: @@ -91,7 +98,7 @@ avoiding large amounts of data being rebalanced. The single-call process unifies exactly what ``prepare`` and ``activate`` do, with the convenience of doing it all at once. Flags and general usage are -equivalent to those of the ``prepare`` subcommand. +equivalent to those of the ``prepare`` and ``activate`` subcommand. **trigger** This subcommand is not meant to be used directly, and it is used by systemd so @@ -136,8 +143,8 @@ group, and lv the logical volume name):: Positional arguments: -* Either in the form of ``vg/lv`` for logical volumes or - ``/path/to/sda1`` for regular devices. +* Either in the form of ``vg/lv`` for logical volumes, + ``/path/to/sda1`` or ``/path/to/sda`` for regular devices. **zap** @@ -157,8 +164,8 @@ Usage, for logical partitions:: Positional arguments: -* Either in the form of ``vg/lv`` for logical volumes or - ``/path/to/sda1`` for regular devices. +* Either in the form of ``vg/lv`` for logical volumes, + ``/path/to/sda1`` or ``/path/to/sda`` for regular devices. simple diff --git a/ceph/doc/man/8/radosgw-admin.rst b/ceph/doc/man/8/radosgw-admin.rst index ab33f7e8c..4aaeb3922 100644 --- a/ceph/doc/man/8/radosgw-admin.rst +++ b/ceph/doc/man/8/radosgw-admin.rst @@ -131,6 +131,12 @@ which are as follows: :command:`zone list` List all zones set on this cluster. +:command:`sync error list` + list sync error. + +:command:`sync error trim` + trim sync error. + :command:`pool add` Add an existing pool for data placement. diff --git a/ceph/doc/mgr/restful.rst b/ceph/doc/mgr/restful.rst index e67f2d144..5ffb16fed 100644 --- a/ceph/doc/mgr/restful.rst +++ b/ceph/doc/mgr/restful.rst @@ -87,3 +87,70 @@ API available via a consistent URL regardless of which manager daemon is currently active, you may want to set up a load balancer front-end to direct traffic to whichever manager endpoint is available. + +Available methods +----------------- + +You can navigate to the ``/doc`` endpoint for full list of available +endpoints and HTTP methods implemented for each endpoint. + +For example, if you want to use the PATCH method of the ``/osd/`` +endpoint to set the state ``up`` of the OSD id ``1``, you can use the +following curl command:: + + echo -En '{"up": true}' | curl --request PATCH --data @- --silent --insecure --user 'https://:/osd/1' + +or you can use python to do so:: + + $ python + >> import requests + >> result = requests.patch( + 'https://:/osd/1', + json={"up": True}, + auth=("", "") + ) + >> print result.json() + +Some of the other endpoints implemented in the *restful* module include + +* ``/config/cluster``: **GET** +* ``/config/osd``: **GET**, **PATCH** +* ``/crush/rule``: **GET** +* ``/mon``: **GET** +* ``/osd``: **GET** +* ``/pool``: **GET**, **POST** +* ``/pool/``: **DELETE**, **GET**, **PATCH** +* ``/request``: **DELETE**, **GET**, **POST** +* ``/request/``: **DELETE**, **GET** +* ``/server``: **GET** + +The ``/request`` endpoint +------------------------- + +You can use the ``/request`` endpoint to poll the state of a request +you scheduled with any **DELETE**, **POST** or **PATCH** method. These +methods are by default asynchronous since it may take longer for them +to finish execution. You can modify this behaviour by appending +``?wait=1`` to the request url. The returned request will then always +be completed. + +The **POST** method of the ``/request`` method provides a passthrough +for the ceph mon commands as defined in ``src/mon/MonCommands.h``. +Let's consider the following command:: + + COMMAND("osd ls " \ + "name=epoch,type=CephInt,range=0,req=false", \ + "show all OSD ids", "osd", "r", "cli,rest") + +The **prefix** is **osd ls**. The optional argument's name is **epoch** +and it is of type ``CephInt``, i.e. ``integer``. This means that you +need to do the following **POST** request to schedule the command:: + + $ python + >> import requests + >> result = requests.post( + 'https://:/request', + json={'prefix': 'osd ls', 'epoch': 0}, + auth=("", "") + ) + >> print result.json() diff --git a/ceph/doc/mgr/zabbix.rst b/ceph/doc/mgr/zabbix.rst index d98540e86..7f4f2d89c 100644 --- a/ceph/doc/mgr/zabbix.rst +++ b/ceph/doc/mgr/zabbix.rst @@ -52,10 +52,10 @@ Restart the ceph-mgr daemon after modifying the setting to load the module. Configuration ------------- -Two configuration keys are mandatory for the module to work: +Two configuration keys are vital for the module to work: -- mgr/zabbix/zabbix_host -- mgr/zabbix/identifier +- zabbix_host +- identifier (optional) The parameter *zabbix_host* controls the hostname of the Zabbix server to which *zabbix_sender* will send the items. This can be a IP-Address if required by @@ -65,6 +65,11 @@ The *identifier* parameter controls the identifier/hostname to use as source when sending items to Zabbix. This should match the name of the *Host* in your Zabbix server. +When the *identifier* parameter is not configured the ceph- of the cluster +will be used when sending data to Zabbix. + +This would for example be *ceph-c4d32a99-9e80-490f-bd3a-1d22d8a7d354* + Additional configuration keys which can be configured and their default values: - mgr/zabbix/zabbix_port: 10051 diff --git a/ceph/doc/rados/command/list-inconsistent-obj.json b/ceph/doc/rados/command/list-inconsistent-obj.json index 859dc84ca..637e3ed8f 100644 --- a/ceph/doc/rados/command/list-inconsistent-obj.json +++ b/ceph/doc/rados/command/list-inconsistent-obj.json @@ -50,7 +50,9 @@ ] }, "selected_object_info": { - "type": "string" + "type": "object", + "description": "Selected object information", + "additionalProperties": true }, "union_shard_errors": { "description": "Union of all shard errors", @@ -60,16 +62,18 @@ "missing", "stat_error", "read_error", - "data_digest_mismatch_oi", - "omap_digest_mismatch_oi", - "size_mismatch_oi", + "data_digest_mismatch_info", + "omap_digest_mismatch_info", + "size_mismatch_info", "ec_hash_error", "ec_size_error", - "oi_attr_missing", - "oi_attr_corrupted", - "obj_size_oi_mismatch", - "ss_attr_missing", - "ss_attr_corrupted" + "info_missing", + "info_corrupted", + "obj_size_info_mismatch", + "snapset_missing", + "snapset_corrupted", + "hinfo_missing", + "hinfo_corrupted" ] }, "minItems": 0, @@ -86,7 +90,8 @@ "size_mismatch", "attr_value_mismatch", "attr_name_mismatch", - "snapset_inconsistency" + "snapset_inconsistency", + "hinfo_inconsistency" ] }, "minItems": 0, @@ -100,7 +105,40 @@ "type": "object", "properties": { "object_info": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "description": "Object information", + "additionalProperties": true + } + ] + }, + "snapset": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "description": "Snap set information", + "additionalProperties": true + } + ] + }, + "hashinfo": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "description": "Erasure code hash information", + "additionalProperties": true + } + ] }, "shard": { "type": "integer" @@ -130,16 +168,18 @@ "missing", "stat_error", "read_error", - "data_digest_mismatch_oi", - "omap_digest_mismatch_oi", - "size_mismatch_oi", + "data_digest_mismatch_info", + "omap_digest_mismatch_info", + "size_mismatch_info", "ec_hash_error", "ec_size_error", - "oi_attr_missing", - "oi_attr_corrupted", - "obj_size_oi_mismatch", - "ss_attr_missing", - "ss_attr_corrupted" + "info_missing", + "info_corrupted", + "obj_size_info_mismatch", + "snapset_missing", + "snapset_corrupted", + "hinfo_missing", + "hinfo_corrupted" ] }, "minItems": 0, @@ -167,11 +207,11 @@ "value", "Base64" ], - "additionalProperties": false, - "minItems": 1 + "additionalProperties": false } } }, + "additionalProperties": false, "required": [ "osd", "primary", diff --git a/ceph/doc/rados/command/list-inconsistent-snap.json b/ceph/doc/rados/command/list-inconsistent-snap.json index 0da6b0f9b..9a463e8ec 100644 --- a/ceph/doc/rados/command/list-inconsistent-snap.json +++ b/ceph/doc/rados/command/list-inconsistent-snap.json @@ -40,11 +40,11 @@ "type": "array", "items": { "enum": [ - "ss_attr_missing", - "ss_attr_corrupted", - "oi_attr_missing", - "oi_attr_corrupted", - "snapset_mismatch", + "snapset_missing", + "snapset_corrupted", + "info_missing", + "info_corrupted", + "snapset_error", "head_mismatch", "headless", "size_mismatch", @@ -52,7 +52,7 @@ "clone_missing" ] }, - "minItems": 1, + "minItems": 0, "uniqueItems": true }, "missing": { diff --git a/ceph/doc/rados/configuration/bluestore-config-ref.rst b/ceph/doc/rados/configuration/bluestore-config-ref.rst index 8d8ace653..542ba151a 100644 --- a/ceph/doc/rados/configuration/bluestore-config-ref.rst +++ b/ceph/doc/rados/configuration/bluestore-config-ref.rst @@ -14,7 +14,7 @@ device. The storage device is normally partitioned into two parts: #. A small partition is formatted with XFS and contains basic metadata for the OSD. This *data directory* includes information about the OSD (its identifier, which cluster it belongs to, and its private - keyring. + keyring). #. The rest of the device is normally a large partition occupying the rest of the device that is managed directly by BlueStore contains @@ -143,8 +143,8 @@ value (usually 4 bytes) for every 4 kilobyte block of data. It is possible to use a smaller checksum value by truncating the checksum to two or one byte, reducing the metadata overhead. The trade-off is that the probability that a random error will not be -detected is higher with a smaller checksum, going from about one if -four billion with a 32-bit (4 byte) checksum to one is 65,536 for a +detected is higher with a smaller checksum, going from about one in +four billion with a 32-bit (4 byte) checksum to one in 65,536 for a 16-bit (2 byte) checksum or one in 256 for an 8-bit (1 byte) checksum. The smaller checksum values can be used by selecting `crc32c_16` or `crc32c_8` as the checksum algorithm. diff --git a/ceph/doc/rados/configuration/storage-devices.rst b/ceph/doc/rados/configuration/storage-devices.rst index 83c0c9b9f..262778d0d 100644 --- a/ceph/doc/rados/configuration/storage-devices.rst +++ b/ceph/doc/rados/configuration/storage-devices.rst @@ -60,7 +60,7 @@ last ten years. Key BlueStore features include: and for erasure coded pools (which rely on cloning to implement efficient two-phase commits). -For more information, see :doc:`bluestore-config-ref`. +For more information, see :doc:`bluestore-config-ref` and :doc:`/rados/operations/bluestore-migration`. FileStore --------- diff --git a/ceph/doc/rados/operations/bluestore-migration.rst b/ceph/doc/rados/operations/bluestore-migration.rst new file mode 100644 index 000000000..f4f16fb71 --- /dev/null +++ b/ceph/doc/rados/operations/bluestore-migration.rst @@ -0,0 +1,292 @@ +===================== + BlueStore Migration +===================== + +Each OSD can run either BlueStore or FileStore, and a single Ceph +cluster can contain a mix of both. Users who have previously deployed +FileStore are likely to want to transition to BlueStore in order to +take advantage of the improved performance and robustness. There are +several strategies for making such a transition. + +An individual OSD cannot be converted in place in isolation, however: +BlueStore and FileStore are simply too different for that to be +practical. "Conversion" will rely either on the cluster's normal +replication and healing support or tools and strategies that copy OSD +content from an old (FileStore) device to a new (BlueStore) one. + + +Deploy new OSDs with BlueStore +============================== + +Any new OSDs (e.g., when the cluster is expanded) can be deployed +using BlueStore. This is the default behavior so no specific change +is needed. + +Similarly, any OSDs that are reprovisioned after replacing a failed drive +can use BlueStore. + +Convert existing OSDs +===================== + +Mark out and replace +-------------------- + +The simplest approach is to mark out each device in turn, wait for the +data to rereplicate across the cluster, reprovision the OSD, and mark +it back in again. It is simple and easy to automate. However, it requires +more data migration than should be necessary, so it is not optimal. + +#. Identify a FileStore OSD to replace:: + + ID= + DEVICE= + + You can tell whether a given OSD is FileStore or BlueStore with:: + + ceph osd metadata $ID | grep osd_objectstore + + You can get a current count of filestore vs bluestore with:: + + ceph osd count-metadata osd_objectstore + +#. Mark the filestore OSD out:: + + ceph osd out $ID + +#. Wait for the data to migrate off the OSD in question:: + + while ! ceph osd safe-to-destroy $ID ; sleep 60 ; done + +#. Stop the OSD:: + + systemctl kill ceph-osd@$ID + +#. Make note of which device this OSD is using:: + + mount | grep /var/lib/ceph/osd/ceph-$ID + +#. Unmount the OSD:: + + umount /var/lib/ceph/osd/ceph-$ID + +#. Destroy the OSD data. Be *EXTREMELY CAREFUL* as this will destroy + the contents of the device; be certain the data on the device is + not needed (i.e., that the cluster is healthy) before proceeding. :: + + ceph-volume lvm zap $DEVICE + +#. Tell the cluster the OSD has been destroyed (and a new OSD can be + reprovisioned with the same ID):: + + ceph osd destroy $ID --yes-i-really-mean-it + +#. Reprovision a BlueStore OSD in its place with the same OSD ID. + This requires you do identify which device to wipe based on what you saw + mounted above. BE CAREFUL! :: + + ceph-volume lvm create --bluestore --data $DEVICE --osd-id $ID + +#. Repeat. + +You can allow the refilling of the replacement OSD to happen +concurrently with the draining of the next OSD, or follow the same +procedure for multiple OSDs in parallel, as long as you ensure the +cluster is fully clean (all data has all replicas) before destroying +any OSDs. Failure to do so will reduce the redundancy of your data +and increase the risk of (or potentially even cause) data loss. + +Advantages: + +* Simple. +* Can be done on a device-by-device basis. +* No spare devices or hosts are required. + +Disadvantages: + +* Data is copied over the network twice: once to some other OSD in the + cluster (to maintain the desired number of replicas), and then again + back to the reprovisioned BlueStore OSD. + + +Whole host replacement +---------------------- + +If you have a spare host in the cluster, or have sufficient free space +to evacuate an entire host in order to use it as a spare, then the +conversion can be done on a host-by-host basis with each stored copy of +the data migrating only once. + +First, you need have empty host that has no data. There are two ways to do this: either by starting with a new, empty host that isn't yet part of the cluster, or by offloading data from an existing host that in the cluster. + +Use a new, empty host +^^^^^^^^^^^^^^^^^^^^^ + +Ideally the host should have roughly the +same capacity as other hosts you will be converting (although it +doesn't strictly matter). :: + + NEWHOST= + +Add the host to the CRUSH hierarchy, but do not attach it to the root:: + + ceph osd crush add-bucket $NEWHOST host + +Make sure the ceph packages are installed. + +Use an existing host +^^^^^^^^^^^^^^^^^^^^ + +If you would like to use an existing host +that is already part of the cluster, and there is sufficient free +space on that host so that all of its data can be migrated off, +then you can instead do:: + + OLDHOST= + ceph osd crush unlink $OLDHOST default + +where "default" is the immediate ancestor in the CRUSH map. (For +smaller clusters with unmodified configurations this will normally +be "default", but it might also be a rack name.) You should now +see the host at the top of the OSD tree output with no parent:: + + $ bin/ceph osd tree + ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF + -5 0 host oldhost + 10 ssd 1.00000 osd.10 up 1.00000 1.00000 + 11 ssd 1.00000 osd.11 up 1.00000 1.00000 + 12 ssd 1.00000 osd.12 up 1.00000 1.00000 + -1 3.00000 root default + -2 3.00000 host foo + 0 ssd 1.00000 osd.0 up 1.00000 1.00000 + 1 ssd 1.00000 osd.1 up 1.00000 1.00000 + 2 ssd 1.00000 osd.2 up 1.00000 1.00000 + ... + +If everything looks good, jump directly to the "Wait for data +migration to complete" step below and proceed from there to clean up +the old OSDs. + +Migration process +^^^^^^^^^^^^^^^^^ + +If you're using a new host, start at step #1. For an existing host, +jump to step #5 below. + +#. Provision new BlueStore OSDs for all devices:: + + ceph-volume lvm create --bluestore --data /dev/$DEVICE + +#. Verify OSDs join the cluster with:: + + ceph osd tree + + You should see the new host ``$NEWHOST`` with all of the OSDs beneath + it, but the host should *not* be nested beneath any other node in + hierarchy (like ``root default``). For example, if ``newhost`` is + the empty host, you might see something like:: + + $ bin/ceph osd tree + ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF + -5 0 host newhost + 10 ssd 1.00000 osd.10 up 1.00000 1.00000 + 11 ssd 1.00000 osd.11 up 1.00000 1.00000 + 12 ssd 1.00000 osd.12 up 1.00000 1.00000 + -1 3.00000 root default + -2 3.00000 host oldhost1 + 0 ssd 1.00000 osd.0 up 1.00000 1.00000 + 1 ssd 1.00000 osd.1 up 1.00000 1.00000 + 2 ssd 1.00000 osd.2 up 1.00000 1.00000 + ... + +#. Identify the first target host to convert :: + + OLDHOST= + +#. Swap the new host into the old host's position in the cluster:: + + ceph osd crush swap-bucket $NEWHOST $OLDHOST + + At this point all data on ``$OLDHOST`` will start migrating to OSDs + on ``$NEWHOST``. If there is a difference in the total capacity of + the old and new hosts you may also see some data migrate to or from + other nodes in the cluster, but as long as the hosts are similarly + sized this will be a relatively small amount of data. + +#. Wait for data migration to complete:: + + while ! ceph osd safe-to-destroy $(ceph osd ls-tree $OLDHOST); do sleep 60 ; done + +#. Stop all old OSDs on the now-empty ``$OLDHOST``:: + + ssh $OLDHOST + systemctl kill ceph-osd.target + umount /var/lib/ceph/osd/ceph-* + +#. Destroy and purge the old OSDs:: + + for osd in `ceph osd ls-tree $OLDHOST`; do + ceph osd purge $osd --yes-i-really-mean-it + done + +#. Wipe the old OSD devices. This requires you do identify which + devices are to be wiped manually (BE CAREFUL!). For each device,:: + + ceph-volume lvm zap $DEVICE + +#. Use the now-empty host as the new host, and repeat:: + + NEWHOST=$OLDHOST + +Advantages: + +* Data is copied over the network only once. +* Converts an entire host's OSDs at once. +* Can parallelize to converting multiple hosts at a time. +* No spare devices are required on each host. + +Disadvantages: + +* A spare host is required. +* An entire host's worth of OSDs will be migrating data at a time. This + is like likely to impact overall cluster performance. +* All migrated data still makes one full hop over the network. + + +Per-OSD device copy +------------------- + +A single logical OSD can be converted by using the ``copy`` function +of ``ceph-objectstore-tool``. This requires that the host have a free +device (or devices) to provision a new, empty BlueStore OSD. For +example, if each host in your cluster has 12 OSDs, then you'd need a +13th available device so that each OSD can be converted in turn before the +old device is reclaimed to convert the next OSD. + +Caveats: + +* This strategy requires that a blank BlueStore OSD be prepared + without allocating a new OSD ID, something that the ``ceph-volume`` + tool doesn't support. More importantly, the setup of *dmcrypt* is + closely tied to the OSD identity, which means that this approach + does not work with encrypted OSDs. + +* The device must be manually partitioned. + +* Tooling not implemented! + +* Not documented! + +Advantages: + +* Little or no data migrates over the network during the conversion. + +Disadvantages: + +* Tooling not fully implemented. +* Process not documented. +* Each host must have a spare or empty device. +* The OSD is offline during the conversion, which means new writes will + be written to only a subset of the OSDs. This increases the risk of data + loss due to a subsequent failure. (However, if there is a failure before + conversion is complete, the original FileStore OSD can be started to provide + access to its original data.) diff --git a/ceph/doc/rados/operations/index.rst b/ceph/doc/rados/operations/index.rst index aacf7648d..253fc2d9d 100644 --- a/ceph/doc/rados/operations/index.rst +++ b/ceph/doc/rados/operations/index.rst @@ -58,6 +58,7 @@ with new hardware. add-or-rm-osds add-or-rm-mons + bluestore-migration Command Reference diff --git a/ceph/doc/rados/operations/pg-states.rst b/ceph/doc/rados/operations/pg-states.rst index 0fbd3dcf0..aa9a3f46b 100644 --- a/ceph/doc/rados/operations/pg-states.rst +++ b/ceph/doc/rados/operations/pg-states.rst @@ -10,6 +10,9 @@ map is ``active + clean``. *Creating* Ceph is still creating the placement group. +*Activating* + The placement group is peered but not yet active. + *Active* Ceph will process requests to the placement group. @@ -41,21 +44,34 @@ map is ``active + clean``. *Forced-Recovery* High recovery priority of that PG is enforced by user. -*Backfill* +*Recovery-wait* + The placement group is waiting in line to start recover. + +*Recovery-toofull* + A recovery operation is waiting because the destination OSD is over its + full ratio. + +*Recovery-unfound* + Recovery stopped due to unfound objects. + +*Backfilling* Ceph is scanning and synchronizing the entire contents of a placement group instead of inferring what contents need to be synchronized from the logs of - recent operations. *Backfill* is a special case of recovery. + recent operations. Backfill is a special case of recovery. *Forced-Backfill* High backfill priority of that PG is enforced by user. -*Wait-backfill* +*Backfill-wait* The placement group is waiting in line to start backfill. *Backfill-toofull* A backfill operation is waiting because the destination OSD is over its full ratio. +*Backfill-unfound* + Backfill stopped due to unfound objects. + *Incomplete* Ceph detects that a placement group is missing information about writes that may have occurred, or does not have any healthy @@ -78,3 +94,12 @@ map is ``active + clean``. The placement group has peered, but cannot serve client IO due to not having enough copies to reach the pool's configured min_size parameter. Recovery may occur in this state, so the pg may heal up to min_size eventually. + +*Snaptrim* + Trimming snaps. + +*Snaptrim-wait* + Queued to trim snaps. + +*Snaptrim-error* + Error stopped trimming snaps. diff --git a/ceph/doc/rados/troubleshooting/troubleshooting-pg.rst b/ceph/doc/rados/troubleshooting/troubleshooting-pg.rst index c51ed316f..eddb9624b 100644 --- a/ceph/doc/rados/troubleshooting/troubleshooting-pg.rst +++ b/ceph/doc/rados/troubleshooting/troubleshooting-pg.rst @@ -383,8 +383,8 @@ objects. If an object named ``foo`` in PG ``0.6`` is truncated, we will have:: "size_mismatch" ], "union_shard_errors": [ - "data_digest_mismatch_oi", - "size_mismatch_oi" + "data_digest_mismatch_info", + "size_mismatch_info" ], "selected_object_info": "0:602f83fe:::foo:head(16'1 client.4110.0:1 dirty|data_digest|omap_digest s 968 uv 1 dd e978e67f od ffffffff alloc_hint [0 0 0])", "shards": [ @@ -405,8 +405,8 @@ objects. If an object named ``foo`` in PG ``0.6`` is truncated, we will have:: { "osd": 2, "errors": [ - "data_digest_mismatch_oi", - "size_mismatch_oi" + "data_digest_mismatch_info", + "size_mismatch_info" ], "size": 0, "omap_digest": "0xffffffff", @@ -437,9 +437,9 @@ In this case, we can learn from the output: ``oi`` indicate a comparison with ``selected_object_info``. Look at the ``shards`` array to determine which shard has which error(s). - * ``data_digest_mismatch_oi``: the digest stored in the object-info is not + * ``data_digest_mismatch_info``: the digest stored in the object-info is not ``0xffffffff``, which is calculated from the shard read from OSD.2 - * ``size_mismatch_oi``: the size stored in the object-info is different + * ``size_mismatch_info``: the size stored in the object-info is different from the one read from OSD.2. The latter is 0. You can repair the inconsistent placement group by executing:: diff --git a/ceph/doc/radosgw/adminops.rst b/ceph/doc/radosgw/adminops.rst index 422dd1652..5da13a8b9 100644 --- a/ceph/doc/radosgw/adminops.rst +++ b/ceph/doc/radosgw/adminops.rst @@ -1909,6 +1909,17 @@ The content must include a JSON representation of the quota settings as encoded in the corresponding read operation. +Set Quota for an Individual Bucket +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To set a quota, the user must have ``buckets`` capability set with ``write`` +permission. :: + + PUT /admin/bucket?quota&uid=&bucket="a + +The content must include a JSON representation of the quota settings +as mentioned in Set Bucket Quota section above. + Standard Error Responses diff --git a/ceph/doc/radosgw/config-ref.rst b/ceph/doc/radosgw/config-ref.rst index ac96193cc..45054a9ec 100644 --- a/ceph/doc/radosgw/config-ref.rst +++ b/ceph/doc/radosgw/config-ref.rst @@ -13,6 +13,17 @@ specified in the command. Thus variables meant to be applied to all RGW instances or all radosgw-admin commands can be put into the ``[global]`` or the ``[client]`` section to avoid specifying instance-name. +``rgw frontends`` + +:Description: Configures the HTTP frontend(s). The configuration for multiple + frontends can be provided in a comma-delimited list. Each frontend + configuration may include a list of options separated by spaces, + where each option is in the form "key=value" or "key". See + `HTTP Frontends`_ for more on supported options. + +:Type: String +:Default: ``civetweb port=7480`` + ``rgw data`` :Description: Sets the location of the data files for Ceph Object Gateway. @@ -848,3 +859,4 @@ Barbican Settings .. _Rados cluster handles: ../../rados/api/librados-intro/#step-2-configuring-a-cluster-handle .. _Barbican: ../barbican .. _Encryption: ../encryption +.. _HTTP Frontends: ../frontends diff --git a/ceph/doc/radosgw/frontends.rst b/ceph/doc/radosgw/frontends.rst new file mode 100644 index 000000000..cdc5c277d --- /dev/null +++ b/ceph/doc/radosgw/frontends.rst @@ -0,0 +1,97 @@ +============== +HTTP Frontends +============== + +.. contents:: + +The Ceph Object Gateway supports two embedded HTTP frontend libraries +that can be configured with ``rgw_frontends``. + +Beast +===== + +.. versionadded:: Luminous + +The ``beast`` frontend uses the Boost.Beast library for HTTP parsing +and the Boost.Asio library for asynchronous network i/o. + +Options +------- + +``port`` + +:Description: Sets the listening port number. + +:Type: Integer +:Default: ``80`` + + +Civetweb +======== + +.. versionadded:: Firefly + +The ``civetweb`` frontend uses the Civetweb HTTP library, which is a +fork of Mongoose. + + +Options +------- + +``port`` + +:Description: Sets the listening port number. For SSL-enabled ports, add an + ``s`` suffix like ``443s``. To bind a specific IPv4 or IPv6 + address, use the form ``address:port``. Multiple endpoints + can be separated by ``+`` as in ``127.0.0.1:8000+443s``. + +:Type: String +:Default: ``7480`` + + +``num_threads`` + +:Description: Sets the number of threads spawned by Civetweb to handle + incoming HTTP connections. This effectively limits the number + of concurrent connections that the frontend can service. + +:Type: Integer +:Default: ``rgw_thread_pool_size`` + + +``request_timeout_ms`` + +:Description: The amount of time in milliseconds that Civetweb will wait + for more incoming data before giving up. + +:Type: Integer +:Default: ``30000`` + + +``ssl_certificate`` + +:Description: Path to the SSL certificate file used for SSL-enabled ports. + +:Type: String +:Default: None + + +A complete list of supported options can be found in the `Civetweb User Manual`_. + + +Generic Options +=============== + +Some frontend options are generic and supported by all frontends: + +``prefix`` + +:Description: A prefix string that is inserted into the URI of all + requests. For example, a swift-only frontend could supply + a uri prefix of ``/swift``. + +:Type: String +:Default: None + + +.. _Civetweb User Manual: https://civetweb.github.io/civetweb/UserManual.html diff --git a/ceph/doc/radosgw/index.rst b/ceph/doc/radosgw/index.rst index 657a6f9b9..b28f76e91 100644 --- a/ceph/doc/radosgw/index.rst +++ b/ceph/doc/radosgw/index.rst @@ -37,6 +37,7 @@ you may write data with one API and retrieve it with the other. :maxdepth: 1 Manual Install w/Civetweb <../../install/install-ceph-gateway> + HTTP Frontends Multisite Configuration Configuring Pools Config Reference diff --git a/ceph/man/conf.py b/ceph/man/conf.py index 67d371e67..c97882529 100644 --- a/ceph/man/conf.py +++ b/ceph/man/conf.py @@ -1,7 +1,7 @@ import os project = u'Ceph' -copyright = u'2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons BY-SA' +copyright = u'2010-2014, Inktank Storage, Inc. and contributors. Licensed under Creative Commons Attribution Share Alike 3.0 (CC-BY-SA-3.0)' version = 'dev' release = 'dev' diff --git a/ceph/qa/cephfs/clusters/1-mds-1-client.yaml b/ceph/qa/cephfs/clusters/1-mds-1-client.yaml new file mode 100644 index 000000000..e64b0b88d --- /dev/null +++ b/ceph/qa/cephfs/clusters/1-mds-1-client.yaml @@ -0,0 +1,8 @@ +roles: +- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3] +- [mon.b, mon.c, mgr.x, mds.a-s, osd.4, osd.5, osd.6, osd.7] +- [client.0] +openstack: +- volumes: # attached to each instance + count: 4 + size: 10 # GB diff --git a/ceph/qa/cephfs/clusters/1-mds-2-client.yaml b/ceph/qa/cephfs/clusters/1-mds-2-client.yaml new file mode 100644 index 000000000..006e15a7b --- /dev/null +++ b/ceph/qa/cephfs/clusters/1-mds-2-client.yaml @@ -0,0 +1,9 @@ +roles: +- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3] +- [mon.b, mon.c, mgr.x, mds.a-s, osd.4, osd.5, osd.6, osd.7] +- [client.0] +- [client.1] +openstack: +- volumes: # attached to each instance + count: 4 + size: 10 # GB diff --git a/ceph/qa/cephfs/clusters/1-mds-4-client.yaml b/ceph/qa/cephfs/clusters/1-mds-4-client.yaml new file mode 100644 index 000000000..a6be36dea --- /dev/null +++ b/ceph/qa/cephfs/clusters/1-mds-4-client.yaml @@ -0,0 +1,11 @@ +roles: +- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3] +- [mon.b, mon.c, mgr.x, mds.a-s, osd.4, osd.5, osd.6, osd.7] +- [client.0] +- [client.1] +- [client.2] +- [client.3] +openstack: +- volumes: # attached to each instance + count: 4 + size: 10 # GB diff --git a/ceph/qa/cephfs/overrides/log-config.yaml b/ceph/qa/cephfs/overrides/log-config.yaml new file mode 100644 index 000000000..3b876f13b --- /dev/null +++ b/ceph/qa/cephfs/overrides/log-config.yaml @@ -0,0 +1,3 @@ +log-rotate: + ceph-mds: 10G + ceph-osd: 10G diff --git a/ceph/qa/cephfs/overrides/osd-asserts.yaml b/ceph/qa/cephfs/overrides/osd-asserts.yaml new file mode 100644 index 000000000..8c16e6e1c --- /dev/null +++ b/ceph/qa/cephfs/overrides/osd-asserts.yaml @@ -0,0 +1,5 @@ +overrides: + ceph: + conf: + osd: + osd shutdown pgref assert: true diff --git a/ceph/qa/standalone/ceph-helpers.sh b/ceph/qa/standalone/ceph-helpers.sh index d7d4db579..6a8eebff4 100755 --- a/ceph/qa/standalone/ceph-helpers.sh +++ b/ceph/qa/standalone/ceph-helpers.sh @@ -1338,6 +1338,8 @@ function test_is_clean() { ####################################################################### +calc() { awk "BEGIN{print $*}"; } + ## # Return a list of numbers that are increasingly larger and whose # total is **timeout** seconds. It can be used to have short sleep @@ -1357,29 +1359,29 @@ function get_timeout_delays() { local i local total="0" i=$first_step - while test "$(echo $total + $i \<= $timeout | bc -l)" = "1"; do - echo -n "$i " - total=$(echo $total + $i | bc -l) - i=$(echo $i \* 2 | bc -l) + while test "$(calc $total + $i \<= $timeout)" = "1"; do + echo -n "$(calc $i) " + total=$(calc $total + $i) + i=$(calc $i \* 2) done - if test "$(echo $total \< $timeout | bc -l)" = "1"; then - echo -n $(echo $timeout - $total | bc -l) + if test "$(calc $total \< $timeout)" = "1"; then + echo -n "$(calc $timeout - $total) " fi $trace && shopt -s -o xtrace } function test_get_timeout_delays() { test "$(get_timeout_delays 1)" = "1 " || return 1 - test "$(get_timeout_delays 5)" = "1 2 2" || return 1 - test "$(get_timeout_delays 6)" = "1 2 3" || return 1 + test "$(get_timeout_delays 5)" = "1 2 2 " || return 1 + test "$(get_timeout_delays 6)" = "1 2 3 " || return 1 test "$(get_timeout_delays 7)" = "1 2 4 " || return 1 - test "$(get_timeout_delays 8)" = "1 2 4 1" || return 1 - test "$(get_timeout_delays 1 .1)" = ".1 .2 .4 .3" || return 1 - test "$(get_timeout_delays 1.5 .1)" = ".1 .2 .4 .8 " || return 1 - test "$(get_timeout_delays 5 .1)" = ".1 .2 .4 .8 1.6 1.9" || return 1 - test "$(get_timeout_delays 6 .1)" = ".1 .2 .4 .8 1.6 2.9" || return 1 - test "$(get_timeout_delays 6.3 .1)" = ".1 .2 .4 .8 1.6 3.2 " || return 1 - test "$(get_timeout_delays 20 .1)" = ".1 .2 .4 .8 1.6 3.2 6.4 7.3" || return 1 + test "$(get_timeout_delays 8)" = "1 2 4 1 " || return 1 + test "$(get_timeout_delays 1 .1)" = "0.1 0.2 0.4 0.3 " || return 1 + test "$(get_timeout_delays 1.5 .1)" = "0.1 0.2 0.4 0.8 " || return 1 + test "$(get_timeout_delays 5 .1)" = "0.1 0.2 0.4 0.8 1.6 1.9 " || return 1 + test "$(get_timeout_delays 6 .1)" = "0.1 0.2 0.4 0.8 1.6 2.9 " || return 1 + test "$(get_timeout_delays 6.3 .1)" = "0.1 0.2 0.4 0.8 1.6 3.2 " || return 1 + test "$(get_timeout_delays 20 .1)" = "0.1 0.2 0.4 0.8 1.6 3.2 6.4 7.3 " || return 1 } ####################################################################### @@ -1734,13 +1736,19 @@ function test_display_logs() { # function run_in_background() { local pid_variable=$1 - shift; + shift # Execute the command and prepend the output with its pid # We enforce to return the exit status of the command and not the awk one. - ("$@" |& awk '{ a[i++] = $0 }END{for (i = 0; i in a; ++i) { print "'$$': " a[i]} }'; return ${PIPESTATUS[0]}) >&2 & + ("$@" |& sed 's/^/'$$': /'; return "${PIPESTATUS[0]}") >&2 & eval "$pid_variable+=\" $!\"" } +function save_stdout { + local out="$1" + shift + "$@" > "$out" +} + function test_run_in_background() { local pids run_in_background pids sleep 1 diff --git a/ceph/qa/standalone/mon/osd-pool-create.sh b/ceph/qa/standalone/mon/osd-pool-create.sh index 693165d89..39eb1c4c3 100755 --- a/ceph/qa/standalone/mon/osd-pool-create.sh +++ b/ceph/qa/standalone/mon/osd-pool-create.sh @@ -22,9 +22,9 @@ function run() { shift export CEPH_MON="127.0.0.1:7105" # git grep '\<7105\>' : there must be only one - export CEPH_ARGS CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " CEPH_ARGS+="--mon-host=$CEPH_MON " + export CEPH_ARGS local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} for func in $funcs ; do @@ -208,6 +208,28 @@ function TEST_utf8_cli() { ceph osd pool delete 黄 黄 --yes-i-really-really-mean-it } +function TEST_pool_create_rep_expected_num_objects() { + local dir=$1 + setup $dir || return 1 + + # disable pg dir merge + CEPH_ARGS+="--filestore-merge-threshold=-10 " + export CEPH_ARGS + run_mon $dir a || return 1 + run_osd $dir 0 || return 1 + + ceph osd pool create rep_expected_num_objects 64 64 replicated replicated_rule 100000 || return 1 + # wait for pg dir creating + sleep 5 + ret=$(find ${dir}/0/current/1.0_head/ | grep DIR | wc -l) + if [ "$ret" -le 2 ]; + then + return 1 + else + echo "TEST_pool_create_rep_expected_num_objects PASS" + fi +} + main osd-pool-create "$@" # Local Variables: diff --git a/ceph/qa/standalone/osd/repro_long_log.sh b/ceph/qa/standalone/osd/repro_long_log.sh new file mode 100755 index 000000000..1f4ba29bd --- /dev/null +++ b/ceph/qa/standalone/osd/repro_long_log.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2014 Cloudwatt +# Copyright (C) 2018 Red Hat +# +# Author: Josh Durgin +# Author: David Zafman +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7100" # git grep '\<7100\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +PGID= + +function test_log_size() +{ + local PGID=$1 + local EXPECTED=$2 + ceph tell osd.\* flush_pg_stats + sleep 3 + ceph pg $PGID query | jq .info.stats.log_size + ceph pg $PGID query | jq .info.stats.log_size | grep "${EXPECTED}" +} + +function setup_log_test() { + local dir=$1 + local which=$2 + + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + run_osd $dir 2 || return 1 + + ceph osd pool create test 1 1 || true + POOL_ID=$(ceph osd dump --format json | jq '.pools[] | select(.pool_name == "test") | .pool') + PGID="${POOL_ID}.0" + + ceph tell osd.\* injectargs -- --osd-min-pg-log-entries 20 || return 1 + ceph tell osd.\* injectargs -- --osd-max-pg-log-entries 30 || return 1 + ceph tell osd.\* injectargs -- --osd-pg-log-trim-min 10 || return 1 + ceph tell osd.\* injectargs -- --osd-pg-log-dups-tracked 10 || return 1 + + touch foo + for i in $(seq 1 20) + do + rados -p test put foo foo || return 1 + done + + test_log_size $PGID 20 || return 1 + + rados -p test rm foo || return 1 + + # generate error entries + for i in $(seq 1 20) + do + rados -p test rm foo + done + + # log should have been trimmed down to min_entries with one extra + test_log_size $PGID 21 || return 1 +} + +function TEST_repro_long_log1() +{ + local dir=$1 + + setup_log_test $dir || return 1 + # regular write should trim the log + rados -p test put foo foo || return 1 + test_log_size $PGID 22 || return 1 +} + +function TEST_repro_long_log2() +{ + local dir=$1 + + setup_log_test $dir || return 1 + local PRIMARY=$(ceph pg $PGID query | jq '.info.stats.up_primary') + kill_daemons $dir TERM osd.$PRIMARY || return 1 + CEPH_ARGS="--osd-max-pg-log-entries=2 --no-mon-config" ceph-objectstore-tool --data-path $dir/$PRIMARY --pgid $PGID --op trim-pg-log || return 1 + run_osd $dir $PRIMARY || return 1 + wait_for_clean || return 1 + test_log_size $PGID 2 || return 1 +} + +function TEST_trim_max_entries() +{ + local dir=$1 + + setup_log_test $dir || return 1 + + ceph tell osd.\* injectargs -- --osd-min-pg-log-entries 1 + ceph tell osd.\* injectargs -- --osd-pg-log-trim-min 2 + ceph tell osd.\* injectargs -- --osd-pg-log-trim-max 4 + + # adding log entries, should only trim 4 and add one each time + rados -p test rm foo + test_log_size $PGID 17 + rados -p test rm foo + test_log_size $PGID 14 + rados -p test rm foo + test_log_size $PGID 11 + rados -p test rm foo + test_log_size $PGID 8 + rados -p test rm foo + test_log_size $PGID 5 + rados -p test rm foo + test_log_size $PGID 2 + + # below trim_min + rados -p test rm foo + test_log_size $PGID 3 + rados -p test rm foo + test_log_size $PGID 4 + + rados -p test rm foo + test_log_size $PGID 2 +} + +main repro-long-log "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && ../qa/run-standalone.sh repro_long_log.sh" +# End: diff --git a/ceph/qa/standalone/scrub/osd-recovery-scrub.sh b/ceph/qa/standalone/scrub/osd-recovery-scrub.sh index ef9a3318a..1061dc77e 100755 --- a/ceph/qa/standalone/scrub/osd-recovery-scrub.sh +++ b/ceph/qa/standalone/scrub/osd-recovery-scrub.sh @@ -65,7 +65,7 @@ function TEST_recovery_scrub() { pids="" for pg in $(seq 0 $(expr $PGS - 1)) do - run_in_background pids pg_scrub $poolid.$(echo "{ obase=16; $pg }" | bc | tr '[:upper:]' '[:lower:]') + run_in_background pids pg_scrub $poolid.$(printf "%x" $pg) done ceph pg dump pgs wait_background pids diff --git a/ceph/qa/standalone/scrub/osd-scrub-repair.sh b/ceph/qa/standalone/scrub/osd-scrub-repair.sh index 557de0d1b..52f171ff1 100755 --- a/ceph/qa/standalone/scrub/osd-scrub-repair.sh +++ b/ceph/qa/standalone/scrub/osd-scrub-repair.sh @@ -29,16 +29,24 @@ fi # Set to "yes" in order to ignore diff errors and save results to update test getjson="no" -# Ignore the epoch and filter out the attr '_' value because it has date information and won't match -if [ "$(jq --version 2>&1 | awk '{ print $3}')" = "1.3" ]; then # Not sure all versions that apply here - jqfilter='.inconsistents | (.[].shards[].attrs[] | select(.name == "_") | .value) |= "----Stripped-by-test----"' -else - jqfilter='.inconsistents | (.[].shards[].attrs[]? | select(.name == "_") | .value) |= "----Stripped-by-test----"' -fi -sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)' +# Filter out mtime and local_mtime dates, version, prior_version and last_reqid (client) from any object_info. +jqfilter='def walk(f): + . as $in + | if type == "object" then + reduce keys[] as $key + ( {}; . + { ($key): ($in[$key] | walk(f)) } ) | f + elif type == "array" then map( walk(f) ) | f + else f + end; +walk(if type == "object" then del(.mtime) else . end) +| walk(if type == "object" then del(.local_mtime) else . end) +| walk(if type == "object" then del(.last_reqid) else . end) +| walk(if type == "object" then del(.version) else . end) +| walk(if type == "object" then del(.prior_version) else . end) +| walk(if type == "object" then del(.redirect_target) else . end) +| walk(if type == "object" then del(.legacy_snaps) else . end)' -# Remove items are not consistent across runs, the pg interval and client -sedfilter='s/\([ ]*\"\(selected_\)*object_info\":.*head[(]\)[^[:space:]]* [^[:space:]]* \(.*\)/\1\3/' +sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)' function run() { local dir=$1 @@ -626,7 +634,7 @@ function TEST_corrupt_scrub_replicated() { # Get epoch for repair-get requests epoch=$(jq .epoch $dir/json) - jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson + jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson { "inconsistents": [ { @@ -638,19 +646,90 @@ function TEST_corrupt_scrub_replicated() { "primary": false }, { + "object_info": { + "oid": { + "oid": "ROBJ1", + "key": "", + "snapid": -2, + "hash": 1454963827, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'58", + "prior_version": "21'3", + "last_reqid": "osd.1.0:57", + "user_version": 3, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xf5fba2c6", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 9, "errors": [ - "size_mismatch_oi", - "obj_size_oi_mismatch" + "size_mismatch_info", + "obj_size_info_mismatch" ], "osd": 1, "primary": true } ], - "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ1", + "key": "", + "snapid": -2, + "hash": 1454963827, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'58", + "prior_version": "21'3", + "last_reqid": "osd.1.0:57", + "user_version": 3, + "size": 7, + "mtime": "2018-04-05 14:33:19.804040", + "local_mtime": "2018-04-05 14:33:19.804839", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xf5fba2c6", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ - "size_mismatch_oi", - "obj_size_oi_mismatch" + "size_mismatch_info", + "obj_size_info_mismatch" ], "errors": [ "size_mismatch" @@ -679,7 +758,42 @@ function TEST_corrupt_scrub_replicated() { "primary": true } ], - "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ12", + "key": "", + "snapid": -2, + "hash": 3920199997, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'56", + "prior_version": "43'36", + "last_reqid": "osd.1.0:55", + "user_version": 36, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x067f306a", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "stat_error" ], @@ -708,7 +822,42 @@ function TEST_corrupt_scrub_replicated() { "primary": true } ], - "selected_object_info": "3:d60617f9:::ROBJ13:head(47'55 osd.0.0:54 dirty|omap|data_digest|omap_digest s 7 uv 39 dd 2ddbf8f5 od 6441854d alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ13", + "key": "", + "snapid": -2, + "hash": 2682806379, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'59", + "prior_version": "45'39", + "last_reqid": "osd.1.0:58", + "user_version": 39, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x6441854d", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "stat_error" ], @@ -724,44 +873,26 @@ function TEST_corrupt_scrub_replicated() { { "shards": [ { + "object_info": "bad-val", "size": 7, - "attrs": [ - { - "Base64": false, - "value": "", - "name": "_" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], "errors": [ - "oi_attr_corrupted" + "info_corrupted" ], "osd": 0, "primary": false }, { "size": 7, - "attrs": [ - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], "errors": [ - "oi_attr_missing" + "info_missing" ], "osd": 1, "primary": true } ], "union_shard_errors": [ - "oi_attr_missing", - "oi_attr_corrupted" + "info_missing", + "info_corrupted" ], "errors": [], "object": { @@ -775,42 +906,94 @@ function TEST_corrupt_scrub_replicated() { { "shards": [ { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" + "object_info": { + "oid": { + "oid": "ROBJ15", + "key": "", + "snapid": -2, + "hash": 504996876, + "max": 0, + "pool": 3, + "namespace": "" }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], + "version": "51'49", + "prior_version": "49'45", + "last_reqid": "osd.1.0:48", + "user_version": 45, + "size": 7, + "mtime": "2018-04-05 14:33:29.498969", + "local_mtime": "2018-04-05 14:33:29.499890", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x2d2a4d6e", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 7, "errors": [], "osd": 0, "primary": false }, { - "attrs": [ - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], "size": 7, "errors": [ - "oi_attr_missing" + "info_missing" ], "osd": 1, "primary": true } ], - "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ15", + "key": "", + "snapid": -2, + "hash": 504996876, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'49", + "prior_version": "49'45", + "last_reqid": "osd.1.0:48", + "user_version": 45, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x2d2a4d6e", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ - "oi_attr_missing" + "info_missing" ], "errors": [], "object": { @@ -832,44 +1015,26 @@ function TEST_corrupt_scrub_replicated() { }, "shards": [ { - "attrs": [ - { - "Base64": true, - "name": "_", - "value": "" - } - ], "errors": [ - "ss_attr_missing" + "snapset_missing" ], "osd": 0, "primary": false, "size": 7 }, { - "attrs": [ - { - "Base64": true, - "name": "_", - "value": "" - }, - { - "Base64": false, - "name": "snapset", - "value": "bad-val" - } - ], "errors": [ - "ss_attr_corrupted" + "snapset_corrupted" ], "osd": 1, "primary": true, + "snapset": "bad-val", "size": 7 } ], "union_shard_errors": [ - "ss_attr_missing", - "ss_attr_corrupted" + "snapset_missing", + "snapset_corrupted" ] }, { @@ -888,7 +1053,42 @@ function TEST_corrupt_scrub_replicated() { "primary": true } ], - "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ3", + "key": "", + "snapid": -2, + "hash": 625845583, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'61", + "prior_version": "25'9", + "last_reqid": "osd.1.0:60", + "user_version": 9, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x00b35dfd", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "missing" ], @@ -905,25 +1105,15 @@ function TEST_corrupt_scrub_replicated() { "shards": [ { "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "bad-val", - "name": "_key1-ROBJ8" + "name": "key1-ROBJ8" }, { "Base64": false, "value": "val2-ROBJ8", - "name": "_key2-ROBJ8" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key2-ROBJ8" } ], "size": 7, @@ -933,25 +1123,15 @@ function TEST_corrupt_scrub_replicated() { }, { "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "val1-ROBJ8", - "name": "_key1-ROBJ8" + "name": "key1-ROBJ8" }, { "Base64": false, "value": "val3-ROBJ8", - "name": "_key3-ROBJ8" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key3-ROBJ8" } ], "size": 7, @@ -960,7 +1140,42 @@ function TEST_corrupt_scrub_replicated() { "primary": true } ], - "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 66 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ8", + "key": "", + "snapid": -2, + "hash": 2359695969, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "79'66", + "prior_version": "79'65", + "last_reqid": "client.4554.0:1", + "user_version": 66, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xd6be81dc", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [], "errors": [ "attr_value_mismatch", @@ -977,25 +1192,130 @@ function TEST_corrupt_scrub_replicated() { { "shards": [ { - "object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 67 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])", + "object_info": { + "oid": { + "oid": "ROBJ9", + "key": "", + "snapid": -2, + "hash": 537189375, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "95'67", + "prior_version": "51'64", + "last_reqid": "client.4649.0:1", + "user_version": 67, + "size": 1, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2b63260d", + "omap_digest": "0x2eecc539", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 1, "errors": [], "osd": 0, "primary": false }, { - "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])", + "object_info": { + "oid": { + "oid": "ROBJ9", + "key": "", + "snapid": -2, + "hash": 537189375, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'64", + "prior_version": "37'27", + "last_reqid": "osd.1.0:63", + "user_version": 27, + "size": 7, + "mtime": "2018-04-05 14:33:25.352485", + "local_mtime": "2018-04-05 14:33:25.353746", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x2eecc539", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 1, "errors": [ - "obj_size_oi_mismatch" + "obj_size_info_mismatch" ], "osd": 1, "primary": true } ], - "selected_object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 67 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ9", + "key": "", + "snapid": -2, + "hash": 537189375, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "95'67", + "prior_version": "51'64", + "last_reqid": "client.4649.0:1", + "user_version": 67, + "size": 1, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2b63260d", + "omap_digest": "0x2eecc539", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ - "obj_size_oi_mismatch" + "obj_size_info_mismatch" ], "errors": [ "object_info_inconsistency" @@ -1013,14 +1333,14 @@ function TEST_corrupt_scrub_replicated() { } EOF - jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson + jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 if test $getjson = "yes" then jq '.' $dir/json > save1.json fi - if which jsonschema > /dev/null; + if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null; then jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 fi @@ -1056,7 +1376,7 @@ EOF # Get epoch for repair-get requests epoch=$(jq .epoch $dir/json) - jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson + jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson { "inconsistents": [ { @@ -1070,23 +1390,94 @@ EOF "primary": false }, { + "object_info": { + "oid": { + "oid": "ROBJ1", + "key": "", + "snapid": -2, + "hash": 1454963827, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'58", + "prior_version": "21'3", + "last_reqid": "osd.1.0:57", + "user_version": 3, + "size": 7, + "mtime": "2018-04-05 14:33:19.804040", + "local_mtime": "2018-04-05 14:33:19.804839", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xf5fba2c6", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "data_digest": "0x2d4a11c2", "omap_digest": "0xf5fba2c6", "size": 9, "errors": [ - "data_digest_mismatch_oi", - "size_mismatch_oi", - "obj_size_oi_mismatch" + "data_digest_mismatch_info", + "size_mismatch_info", + "obj_size_info_mismatch" ], "osd": 1, "primary": true } ], - "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ1", + "key": "", + "snapid": -2, + "hash": 1454963827, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'58", + "prior_version": "21'3", + "last_reqid": "osd.1.0:57", + "user_version": 3, + "size": 7, + "mtime": "2018-04-05 14:33:19.804040", + "local_mtime": "2018-04-05 14:33:19.804839", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xf5fba2c6", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ - "data_digest_mismatch_oi", - "size_mismatch_oi", - "obj_size_oi_mismatch" + "data_digest_mismatch_info", + "size_mismatch_info", + "obj_size_info_mismatch" ], "errors": [ "data_digest_mismatch", @@ -1107,7 +1498,7 @@ EOF "omap_digest": "0xa8dd5adc", "size": 7, "errors": [ - "omap_digest_mismatch_oi" + "omap_digest_mismatch_info" ], "osd": 0, "primary": false @@ -1117,15 +1508,50 @@ EOF "omap_digest": "0xa8dd5adc", "size": 7, "errors": [ - "omap_digest_mismatch_oi" + "omap_digest_mismatch_info" ], "osd": 1, "primary": true } ], - "selected_object_info": "3:b1f19cbd:::ROBJ10:head(47'51 osd.0.0:50 dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [0 0 0])", + "selected_object_info": { + "alloc_hint_flags": 0, + "data_digest": "0x2ddbf8f5", + "expected_object_size": 0, + "expected_write_size": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "last_reqid": "osd.0.0:50", + "local_mtime": "2018-04-05 14:33:26.762368", + "lost": 0, + "manifest": { + "type": 0 + }, + "mtime": "2018-04-05 14:33:26.762368", + "oid": { + "hash": 3174666125, + "key": "", + "max": 0, + "namespace": "", + "oid": "ROBJ10", + "pool": 3, + "snapid": -2 + }, + "omap_digest": "0xc2025a24", + "prior_version": "41'33", + "size": 7, + "truncate_seq": 0, + "truncate_size": 0, + "user_version": 30, + "version": "47'51", + "watchers": {} + }, "union_shard_errors": [ - "omap_digest_mismatch_oi" + "omap_digest_mismatch_info" ], "errors": [], "object": { @@ -1155,7 +1581,42 @@ EOF "primary": true } ], - "selected_object_info": "3:87abbf36:::ROBJ11:head(47'48 osd.0.0:47 dirty|omap|data_digest|omap_digest s 7 uv 33 dd 2ddbf8f5 od a03cef03 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ11", + "key": "", + "snapid": -2, + "hash": 1828574689, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'52", + "prior_version": "41'33", + "last_reqid": "osd.1.0:51", + "user_version": 33, + "size": 7, + "mtime": "2018-04-05 14:33:26.761286", + "local_mtime": "2018-04-05 14:33:26.762368", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xa03cef03", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "read_error" ], @@ -1186,7 +1647,42 @@ EOF "primary": true } ], - "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ12", + "key": "", + "snapid": -2, + "hash": 3920199997, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'56", + "prior_version": "43'36", + "last_reqid": "osd.1.0:55", + "user_version": 36, + "size": 7, + "mtime": "2018-04-05 14:33:27.460958", + "local_mtime": "2018-04-05 14:33:27.462109", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x067f306a", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "stat_error" ], @@ -1233,48 +1729,30 @@ EOF { "shards": [ { - "attrs": [ - { - "Base64": false, - "value": "", - "name": "_" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], + "object_info": "bad-val", "data_digest": "0x2ddbf8f5", "omap_digest": "0x4f14f849", "size": 7, "errors": [ - "oi_attr_corrupted" + "info_corrupted" ], "osd": 0, "primary": false }, { - "attrs": [ - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], "data_digest": "0x2ddbf8f5", "omap_digest": "0x4f14f849", "size": 7, "errors": [ - "oi_attr_missing" + "info_missing" ], "osd": 1, "primary": true } ], "union_shard_errors": [ - "oi_attr_missing", - "oi_attr_corrupted" + "info_missing", + "info_corrupted" ], "errors": [], "object": { @@ -1288,18 +1766,42 @@ EOF { "shards": [ { - "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" + "object_info": { + "oid": { + "oid": "ROBJ15", + "key": "", + "snapid": -2, + "hash": 504996876, + "max": 0, + "pool": 3, + "namespace": "" }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], + "version": "51'49", + "prior_version": "49'45", + "last_reqid": "osd.1.0:48", + "user_version": 45, + "size": 7, + "mtime": "2018-04-05 14:33:29.498969", + "local_mtime": "2018-04-05 14:33:29.499890", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x2d2a4d6e", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "data_digest": "0x2ddbf8f5", "omap_digest": "0x2d2a4d6e", "size": 7, @@ -1308,26 +1810,54 @@ EOF "primary": false }, { - "attrs": [ - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" - } - ], "data_digest": "0x2ddbf8f5", "omap_digest": "0x2d2a4d6e", "size": 7, "errors": [ - "oi_attr_missing" + "info_missing" ], "osd": 1, "primary": true } ], - "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ15", + "key": "", + "snapid": -2, + "hash": 504996876, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'49", + "prior_version": "49'45", + "last_reqid": "osd.1.0:48", + "user_version": 45, + "size": 7, + "mtime": "2018-04-05 14:33:29.498969", + "local_mtime": "2018-04-05 14:33:29.499890", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x2d2a4d6e", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ - "oi_attr_missing" + "info_missing" ], "errors": [], "object": { @@ -1349,16 +1879,9 @@ EOF }, "shards": [ { - "attrs": [ - { - "Base64": true, - "name": "_", - "value": "" - } - ], "data_digest": "0x2ddbf8f5", "errors": [ - "ss_attr_missing" + "snapset_missing" ], "omap_digest": "0x8b699207", "osd": 0, @@ -1366,21 +1889,10 @@ EOF "size": 7 }, { - "attrs": [ - { - "Base64": true, - "name": "_", - "value": "" - }, - { - "Base64": false, - "name": "snapset", - "value": "bad-val" - } - ], + "snapset": "bad-val", "data_digest": "0x2ddbf8f5", "errors": [ - "ss_attr_corrupted" + "snapset_corrupted" ], "omap_digest": "0x8b699207", "osd": 1, @@ -1389,8 +1901,8 @@ EOF } ], "union_shard_errors": [ - "ss_attr_missing", - "ss_attr_corrupted" + "snapset_missing", + "snapset_corrupted" ] }, { @@ -1400,7 +1912,7 @@ EOF "omap_digest": "0xf8e11918", "size": 7, "errors": [ - "data_digest_mismatch_oi" + "data_digest_mismatch_info" ], "osd": 0, "primary": false @@ -1414,9 +1926,44 @@ EOF "primary": true } ], - "selected_object_info": "3:e97ce31e:::ROBJ2:head(47'56 osd.0.0:55 dirty|omap|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od f8e11918 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ2", + "key": "", + "snapid": -2, + "hash": 2026323607, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'60", + "prior_version": "23'6", + "last_reqid": "osd.1.0:59", + "user_version": 6, + "size": 7, + "mtime": "2018-04-05 14:33:20.498756", + "local_mtime": "2018-04-05 14:33:20.499704", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xf8e11918", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ - "data_digest_mismatch_oi" + "data_digest_mismatch_info" ], "errors": [ "data_digest_mismatch" @@ -1447,7 +1994,42 @@ EOF "primary": true } ], - "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ3", + "key": "", + "snapid": -2, + "hash": 625845583, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'61", + "prior_version": "25'9", + "last_reqid": "osd.1.0:60", + "user_version": 9, + "size": 7, + "mtime": "2018-04-05 14:33:21.189382", + "local_mtime": "2018-04-05 14:33:21.190446", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x00b35dfd", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "missing" ], @@ -1467,7 +2049,7 @@ EOF "omap_digest": "0xd7178dfe", "size": 7, "errors": [ - "omap_digest_mismatch_oi" + "omap_digest_mismatch_info" ], "osd": 0, "primary": false @@ -1481,9 +2063,44 @@ EOF "primary": true } ], - "selected_object_info": "3:f4981d31:::ROBJ4:head(47'58 osd.0.0:57 dirty|omap|data_digest|omap_digest s 7 uv 12 dd 2ddbf8f5 od e2d46ea4 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ4", + "key": "", + "snapid": -2, + "hash": 2360875311, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'62", + "prior_version": "27'12", + "last_reqid": "osd.1.0:61", + "user_version": 12, + "size": 7, + "mtime": "2018-04-05 14:33:21.862313", + "local_mtime": "2018-04-05 14:33:21.863261", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xe2d46ea4", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ - "omap_digest_mismatch_oi" + "omap_digest_mismatch_info" ], "errors": [ "omap_digest_mismatch" @@ -1511,15 +2128,50 @@ EOF "omap_digest": "0x06cac8f6", "size": 7, "errors": [ - "omap_digest_mismatch_oi" + "omap_digest_mismatch_info" ], "osd": 1, "primary": true } ], - "selected_object_info": "3:f4bfd4d1:::ROBJ5:head(47'59 osd.0.0:58 dirty|omap|data_digest|omap_digest s 7 uv 15 dd 2ddbf8f5 od 1a862a41 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ5", + "key": "", + "snapid": -2, + "hash": 2334915887, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'63", + "prior_version": "29'15", + "last_reqid": "osd.1.0:62", + "user_version": 15, + "size": 7, + "mtime": "2018-04-05 14:33:22.589300", + "local_mtime": "2018-04-05 14:33:22.590376", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x1a862a41", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ - "omap_digest_mismatch_oi" + "omap_digest_mismatch_info" ], "errors": [ "omap_digest_mismatch" @@ -1539,7 +2191,7 @@ EOF "omap_digest": "0x689ee887", "size": 7, "errors": [ - "omap_digest_mismatch_oi" + "omap_digest_mismatch_info" ], "osd": 0, "primary": false @@ -1553,9 +2205,44 @@ EOF "primary": true } ], - "selected_object_info": "3:a53c12e8:::ROBJ6:head(47'50 osd.0.0:49 dirty|omap|data_digest|omap_digest s 7 uv 18 dd 2ddbf8f5 od 179c919f alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ6", + "key": "", + "snapid": -2, + "hash": 390610085, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'54", + "prior_version": "31'18", + "last_reqid": "osd.1.0:53", + "user_version": 18, + "size": 7, + "mtime": "2018-04-05 14:33:23.289188", + "local_mtime": "2018-04-05 14:33:23.290130", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x179c919f", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ - "omap_digest_mismatch_oi" + "omap_digest_mismatch_info" ], "errors": [ "omap_digest_mismatch" @@ -1583,15 +2270,50 @@ EOF "omap_digest": "0x6a73cc07", "size": 7, "errors": [ - "omap_digest_mismatch_oi" + "omap_digest_mismatch_info" ], "osd": 1, "primary": true } ], - "selected_object_info": "3:8b55fa4b:::ROBJ7:head(47'49 osd.0.0:48 dirty|omap|data_digest|omap_digest s 7 uv 21 dd 2ddbf8f5 od efced57a alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ7", + "key": "", + "snapid": -2, + "hash": 3529485009, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'53", + "prior_version": "33'21", + "last_reqid": "osd.1.0:52", + "user_version": 21, + "size": 7, + "mtime": "2018-04-05 14:33:23.979658", + "local_mtime": "2018-04-05 14:33:23.980731", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xefced57a", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ - "omap_digest_mismatch_oi" + "omap_digest_mismatch_info" ], "errors": [ "omap_digest_mismatch" @@ -1608,25 +2330,15 @@ EOF "shards": [ { "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "bad-val", - "name": "_key1-ROBJ8" + "name": "key1-ROBJ8" }, { "Base64": false, "value": "val2-ROBJ8", - "name": "_key2-ROBJ8" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key2-ROBJ8" } ], "data_digest": "0x2ddbf8f5", @@ -1638,25 +2350,15 @@ EOF }, { "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "val1-ROBJ8", - "name": "_key1-ROBJ8" + "name": "key1-ROBJ8" }, { "Base64": false, "value": "val3-ROBJ8", - "name": "_key3-ROBJ8" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key3-ROBJ8" } ], "data_digest": "0x2ddbf8f5", @@ -1667,7 +2369,42 @@ EOF "primary": true } ], - "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 66 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ8", + "key": "", + "snapid": -2, + "hash": 2359695969, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "79'66", + "prior_version": "79'65", + "last_reqid": "client.4554.0:1", + "user_version": 66, + "size": 7, + "mtime": "2018-04-05 14:34:05.598688", + "local_mtime": "2018-04-05 14:34:05.599698", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xd6be81dc", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [], "errors": [ "attr_value_mismatch", @@ -1684,18 +2421,88 @@ EOF { "shards": [ { - "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])", + "object_info": { + "oid": { + "oid": "ROBJ9", + "key": "", + "snapid": -2, + "hash": 537189375, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "51'64", + "prior_version": "37'27", + "last_reqid": "osd.1.0:63", + "user_version": 27, + "size": 7, + "mtime": "2018-04-05 14:33:25.352485", + "local_mtime": "2018-04-05 14:33:25.353746", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0x2eecc539", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "data_digest": "0x1f26fb26", "omap_digest": "0x2eecc539", "size": 3, "errors": [ - "obj_size_oi_mismatch" + "obj_size_info_mismatch" ], "osd": 0, "primary": false }, { - "object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 68 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])", + "object_info": { + "oid": { + "oid": "ROBJ9", + "key": "", + "snapid": -2, + "hash": 537189375, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "119'68", + "prior_version": "51'64", + "last_reqid": "client.4834.0:1", + "user_version": 68, + "size": 3, + "mtime": "2018-04-05 14:35:01.500659", + "local_mtime": "2018-04-05 14:35:01.502117", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x1f26fb26", + "omap_digest": "0x2eecc539", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "data_digest": "0x1f26fb26", "omap_digest": "0x2eecc539", "size": 3, @@ -1704,9 +2511,44 @@ EOF "primary": true } ], - "selected_object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 68 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ9", + "key": "", + "snapid": -2, + "hash": 537189375, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "119'68", + "prior_version": "51'64", + "last_reqid": "client.4834.0:1", + "user_version": 68, + "size": 3, + "mtime": "2018-04-05 14:35:01.500659", + "local_mtime": "2018-04-05 14:35:01.502117", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x1f26fb26", + "omap_digest": "0x2eecc539", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ - "obj_size_oi_mismatch" + "obj_size_info_mismatch" ], "errors": [ "object_info_inconsistency" @@ -1724,14 +2566,14 @@ EOF } EOF - jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson + jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 if test $getjson = "yes" then jq '.' $dir/json > save2.json fi - if which jsonschema > /dev/null; + if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null; then jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 fi @@ -1748,7 +2590,7 @@ function corrupt_scrub_erasure() { local dir=$1 local allow_overwrites=$2 local poolname=ecpool - local total_objs=5 + local total_objs=7 setup $dir || return 1 run_mon $dir a || return 1 @@ -1810,6 +2652,23 @@ function corrupt_scrub_erasure() { objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 ;; + 6) + objectstore_tool $dir 0 $objname rm-attr hinfo_key || return 1 + echo -n bad-val > $dir/bad-val + objectstore_tool $dir 1 $objname set-attr hinfo_key $dir/bad-val || return 1 + ;; + + 7) + local payload=MAKETHISDIFFERENTFROMOTHEROBJECTS + echo $payload > $dir/DIFFERENT + rados --pool $poolname put $objname $dir/DIFFERENT || return 1 + + # Get hinfo_key from EOBJ1 + objectstore_tool $dir 0 EOBJ1 get-attr hinfo_key > $dir/hinfo + objectstore_tool $dir 0 $objname set-attr hinfo_key $dir/hinfo || return 1 + rm -f $dir/hinfo + ;; + esac done @@ -1827,7 +2686,7 @@ function corrupt_scrub_erasure() { # Get epoch for repair-get requests epoch=$(jq .epoch $dir/json) - jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson + jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson { "inconsistents": [ { @@ -1840,11 +2699,46 @@ function corrupt_scrub_erasure() { "primary": false }, { + "object_info": { + "oid": { + "oid": "EOBJ1", + "key": "", + "snapid": -2, + "hash": 560836233, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "27'1", + "prior_version": "0'0", + "last_reqid": "client.4184.0:1", + "user_version": 1, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 9, "shard": 0, "errors": [ - "size_mismatch_oi", - "obj_size_oi_mismatch" + "size_mismatch_info", + "obj_size_info_mismatch" ], "osd": 1, "primary": true @@ -1857,10 +2751,44 @@ function corrupt_scrub_erasure() { "primary": false } ], - "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ1", + "key": "", + "snapid": -2, + "hash": 560836233, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "27'1", + "prior_version": "0'0", + "last_reqid": "client.4184.0:1", + "user_version": 1, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ - "size_mismatch_oi", - "obj_size_oi_mismatch" + "size_mismatch_info", + "obj_size_info_mismatch" ], "errors": [ "size_mismatch" @@ -1898,7 +2826,41 @@ function corrupt_scrub_erasure() { "primary": false } ], - "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ3", + "key": "", + "snapid": -2, + "hash": 3125668237, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "39'3", + "prior_version": "0'0", + "last_reqid": "client.4252.0:1", + "user_version": 3, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "missing" ], @@ -1915,30 +2877,15 @@ function corrupt_scrub_erasure() { "shards": [ { "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "bad-val", - "name": "_key1-EOBJ4" + "name": "key1-EOBJ4" }, { "Base64": false, "value": "val2-EOBJ4", - "name": "_key2-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key2-EOBJ4" } ], "size": 2048, @@ -1954,30 +2901,15 @@ function corrupt_scrub_erasure() { "errors": [], "size": 2048, "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "val1-EOBJ4", - "name": "_key1-EOBJ4" + "name": "key1-EOBJ4" }, { "Base64": false, "value": "val2-EOBJ4", - "name": "_key2-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key2-EOBJ4" } ] }, @@ -1988,35 +2920,54 @@ function corrupt_scrub_erasure() { "errors": [], "size": 2048, "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "val1-EOBJ4", - "name": "_key1-EOBJ4" + "name": "key1-EOBJ4" }, { "Base64": false, "value": "val3-EOBJ4", - "name": "_key3-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key3-EOBJ4" } ] } ], - "selected_object_info": "3:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ4", + "key": "", + "snapid": -2, + "hash": 1618759290, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "45'6", + "prior_version": "45'5", + "last_reqid": "client.4294.0:1", + "user_version": 6, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [], "errors": [ "attr_value_mismatch", @@ -2040,11 +2991,46 @@ function corrupt_scrub_erasure() { "primary": false }, { + "object_info": { + "oid": { + "oid": "EOBJ5", + "key": "", + "snapid": -2, + "hash": 2918945441, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "59'7", + "prior_version": "0'0", + "last_reqid": "client.4382.0:1", + "user_version": 7, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 4096, "shard": 0, "errors": [ - "size_mismatch_oi", - "obj_size_oi_mismatch" + "size_mismatch_info", + "obj_size_info_mismatch" ], "osd": 1, "primary": true @@ -2057,10 +3043,44 @@ function corrupt_scrub_erasure() { "primary": false } ], - "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ5", + "key": "", + "snapid": -2, + "hash": 2918945441, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "59'7", + "prior_version": "0'0", + "last_reqid": "client.4382.0:1", + "user_version": 7, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ - "size_mismatch_oi", - "obj_size_oi_mismatch" + "size_mismatch_info", + "obj_size_info_mismatch" ], "errors": [ "size_mismatch" @@ -2072,26 +3092,242 @@ function corrupt_scrub_erasure() { "nspace": "", "name": "EOBJ5" } - } - ], - "epoch": 0 -} -EOF - - jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson - diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 - if test $getjson = "yes" - then - jq '.' $dir/json > save3.json - fi - - if which jsonschema > /dev/null; - then - jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 - fi - - pg_deep_scrub $pg - + }, + { + "errors": [], + "object": { + "locator": "", + "name": "EOBJ6", + "nspace": "", + "snap": "head", + "version": 8 + }, + "selected_object_info": { + "oid": { + "oid": "EOBJ6", + "key": "", + "snapid": -2, + "hash": 3050890866, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "65'8", + "prior_version": "0'0", + "last_reqid": "client.4418.0:1", + "user_version": 8, + "size": 7, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, + "shards": [ + { + "errors": [ + "hinfo_missing" + ], + "osd": 0, + "primary": false, + "shard": 2, + "size": 2048 + }, + { + "errors": [ + "hinfo_corrupted" + ], + "osd": 1, + "primary": true, + "shard": 0, + "hashinfo": "bad-val", + "size": 2048 + }, + { + "errors": [], + "osd": 2, + "primary": false, + "shard": 1, + "size": 2048, + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 80717615, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 80717615, + "shard": 2 + } + ], + "total_chunk_size": 2048 + } + } + ], + "union_shard_errors": [ + "hinfo_missing", + "hinfo_corrupted" + ] + }, + { + "errors": [ + "hinfo_inconsistency" + ], + "object": { + "locator": "", + "name": "EOBJ7", + "nspace": "", + "snap": "head", + "version": 10 + }, + "selected_object_info": { + "oid": { + "oid": "EOBJ7", + "key": "", + "snapid": -2, + "hash": 3258066308, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "75'10", + "prior_version": "75'9", + "last_reqid": "client.4482.0:1", + "user_version": 10, + "size": 34, + "mtime": "", + "local_mtime": "", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x136e4e27", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, + "shards": [ + { + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 80717615, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 80717615, + "shard": 2 + } + ], + "total_chunk_size": 2048 + }, + "errors": [], + "osd": 0, + "primary": false, + "shard": 2, + "size": 2048 + }, + { + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 1534350760, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 1534350760, + "shard": 2 + } + ], + "total_chunk_size": 2048 + }, + "errors": [], + "osd": 1, + "primary": true, + "shard": 0, + "size": 2048 + }, + { + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 1534350760, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 1534350760, + "shard": 2 + } + ], + "total_chunk_size": 2048 + }, + "errors": [], + "osd": 2, + "primary": false, + "shard": 1, + "size": 2048 + } + ], + "union_shard_errors": [] + } + ], + "epoch": 0 +} +EOF + + jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson + diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 + if test $getjson = "yes" + then + jq '.' $dir/json > save3.json + fi + + if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null; + then + jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 + fi + + pg_deep_scrub $pg + rados list-inconsistent-pg $poolname > $dir/json || return 1 # Check pg count test $(jq '. | length' $dir/json) = "1" || return 1 @@ -2104,7 +3340,7 @@ EOF if [ "$allow_overwrites" = "true" ] then - jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson + jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson { "inconsistents": [ { @@ -2119,12 +3355,47 @@ EOF "primary": false }, { + "object_info": { + "oid": { + "oid": "EOBJ1", + "key": "", + "snapid": -2, + "hash": 560836233, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "27'1", + "prior_version": "0'0", + "last_reqid": "client.4184.0:1", + "user_version": 1, + "size": 7, + "mtime": "2018-04-05 14:31:33.837147", + "local_mtime": "2018-04-05 14:31:33.840763", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 9, "shard": 0, "errors": [ "read_error", - "size_mismatch_oi", - "obj_size_oi_mismatch" + "size_mismatch_info", + "obj_size_info_mismatch" ], "osd": 1, "primary": true @@ -2139,11 +3410,45 @@ EOF "primary": false } ], - "selected_object_info": "3:9175b684:::EOBJ1:head(27'1 client.4155.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ1", + "key": "", + "snapid": -2, + "hash": 560836233, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "27'1", + "prior_version": "0'0", + "last_reqid": "client.4184.0:1", + "user_version": 1, + "size": 7, + "mtime": "2018-04-05 14:31:33.837147", + "local_mtime": "2018-04-05 14:31:33.840763", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "read_error", - "size_mismatch_oi", - "obj_size_oi_mismatch" + "size_mismatch_info", + "obj_size_info_mismatch" ], "errors": [ "size_mismatch" @@ -2185,7 +3490,41 @@ EOF "primary": false } ], - "selected_object_info": "3:b197b25d:::EOBJ3:head(41'3 client.4199.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ3", + "key": "", + "snapid": -2, + "hash": 3125668237, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "39'3", + "prior_version": "0'0", + "last_reqid": "client.4252.0:1", + "user_version": 3, + "size": 7, + "mtime": "2018-04-05 14:31:46.841145", + "local_mtime": "2018-04-05 14:31:46.844996", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "missing" ], @@ -2202,30 +3541,15 @@ EOF "shards": [ { "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "bad-val", - "name": "_key1-EOBJ4" + "name": "key1-EOBJ4" }, { "Base64": false, "value": "val2-EOBJ4", - "name": "_key2-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key2-EOBJ4" } ], "data_digest": "0x00000000", @@ -2238,30 +3562,15 @@ EOF }, { "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "val1-EOBJ4", - "name": "_key1-EOBJ4" + "name": "key1-EOBJ4" }, { "Base64": false, "value": "val2-EOBJ4", - "name": "_key2-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key2-EOBJ4" } ], "data_digest": "0x00000000", @@ -2274,30 +3583,15 @@ EOF }, { "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "val1-EOBJ4", - "name": "_key1-EOBJ4" + "name": "key1-EOBJ4" }, { "Base64": false, "value": "val3-EOBJ4", - "name": "_key3-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key3-EOBJ4" } ], "data_digest": "0x00000000", @@ -2309,7 +3603,41 @@ EOF "primary": false } ], - "selected_object_info": "3:5e723e06:::EOBJ4:head(48'6 client.4223.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ4", + "key": "", + "snapid": -2, + "hash": 1618759290, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "45'6", + "prior_version": "45'5", + "last_reqid": "client.4294.0:1", + "user_version": 6, + "size": 7, + "mtime": "2018-04-05 14:31:54.663622", + "local_mtime": "2018-04-05 14:31:54.664527", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [], "errors": [ "attr_value_mismatch", @@ -2337,10 +3665,45 @@ EOF { "data_digest": "0x00000000", "omap_digest": "0xffffffff", + "object_info": { + "oid": { + "oid": "EOBJ5", + "key": "", + "snapid": -2, + "hash": 2918945441, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "59'7", + "prior_version": "0'0", + "last_reqid": "client.4382.0:1", + "user_version": 7, + "size": 7, + "mtime": "2018-04-05 14:32:12.929161", + "local_mtime": "2018-04-05 14:32:12.934707", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 4096, "errors": [ - "size_mismatch_oi", - "obj_size_oi_mismatch" + "size_mismatch_info", + "obj_size_info_mismatch" ], "shard": 0, "osd": 1, @@ -2356,10 +3719,44 @@ EOF "primary": false } ], - "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4288.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ5", + "key": "", + "snapid": -2, + "hash": 2918945441, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "59'7", + "prior_version": "0'0", + "last_reqid": "client.4382.0:1", + "user_version": 7, + "size": 7, + "mtime": "2018-04-05 14:32:12.929161", + "local_mtime": "2018-04-05 14:32:12.934707", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ - "size_mismatch_oi", - "obj_size_oi_mismatch" + "size_mismatch_info", + "obj_size_info_mismatch" ], "errors": [ "size_mismatch" @@ -2371,6 +3768,233 @@ EOF "nspace": "", "name": "EOBJ5" } + }, + { + "object": { + "name": "EOBJ6", + "nspace": "", + "locator": "", + "snap": "head", + "version": 8 + }, + "errors": [], + "union_shard_errors": [ + "read_error", + "hinfo_missing", + "hinfo_corrupted" + ], + "selected_object_info": { + "oid": { + "oid": "EOBJ6", + "key": "", + "snapid": -2, + "hash": 3050890866, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "65'8", + "prior_version": "0'0", + "last_reqid": "client.4418.0:1", + "user_version": 8, + "size": 7, + "mtime": "2018-04-05 14:32:20.634116", + "local_mtime": "2018-04-05 14:32:20.637999", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, + "shards": [ + { + "osd": 0, + "primary": false, + "shard": 2, + "errors": [ + "read_error", + "hinfo_missing" + ], + "size": 2048 + }, + { + "osd": 1, + "primary": true, + "shard": 0, + "errors": [ + "read_error", + "hinfo_corrupted" + ], + "size": 2048, + "hashinfo": "bad-val" + }, + { + "osd": 2, + "primary": false, + "shard": 1, + "errors": [], + "size": 2048, + "omap_digest": "0xffffffff", + "data_digest": "0x00000000", + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 80717615, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 80717615, + "shard": 2 + } + ], + "total_chunk_size": 2048 + } + } + ] + }, + { + "object": { + "name": "EOBJ7", + "nspace": "", + "locator": "", + "snap": "head", + "version": 10 + }, + "errors": [ + "hinfo_inconsistency" + ], + "union_shard_errors": [], + "selected_object_info": { + "oid": { + "oid": "EOBJ7", + "key": "", + "snapid": -2, + "hash": 3258066308, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "75'10", + "prior_version": "75'9", + "last_reqid": "client.4482.0:1", + "user_version": 10, + "size": 34, + "mtime": "2018-04-05 14:32:33.058782", + "local_mtime": "2018-04-05 14:32:33.059679", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x136e4e27", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, + "shards": [ + { + "osd": 0, + "primary": false, + "shard": 2, + "errors": [], + "size": 2048, + "omap_digest": "0xffffffff", + "data_digest": "0x00000000", + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 80717615, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 80717615, + "shard": 2 + } + ], + "total_chunk_size": 2048 + } + }, + { + "osd": 1, + "primary": true, + "shard": 0, + "errors": [], + "size": 2048, + "omap_digest": "0xffffffff", + "data_digest": "0x00000000", + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 1534350760, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 1534350760, + "shard": 2 + } + ], + "total_chunk_size": 2048 + } + }, + { + "osd": 2, + "primary": false, + "shard": 1, + "errors": [], + "size": 2048, + "omap_digest": "0xffffffff", + "data_digest": "0x00000000", + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 1534350760, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 1534350760, + "shard": 2 + } + ], + "total_chunk_size": 2048 + } + } + ] } ], "epoch": 0 @@ -2379,7 +4003,7 @@ EOF else - jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson + jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson { "inconsistents": [ { @@ -2394,12 +4018,47 @@ EOF "primary": false }, { + "object_info": { + "oid": { + "oid": "EOBJ1", + "key": "", + "snapid": -2, + "hash": 560836233, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "27'1", + "prior_version": "0'0", + "last_reqid": "client.4192.0:1", + "user_version": 1, + "size": 7, + "mtime": "2018-04-05 14:30:10.688009", + "local_mtime": "2018-04-05 14:30:10.691774", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 9, "shard": 0, "errors": [ "read_error", - "size_mismatch_oi", - "obj_size_oi_mismatch" + "size_mismatch_info", + "obj_size_info_mismatch" ], "osd": 1, "primary": true @@ -2414,11 +4073,45 @@ EOF "primary": false } ], - "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ1", + "key": "", + "snapid": -2, + "hash": 560836233, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "27'1", + "prior_version": "0'0", + "last_reqid": "client.4192.0:1", + "user_version": 1, + "size": 7, + "mtime": "2018-04-05 14:30:10.688009", + "local_mtime": "2018-04-05 14:30:10.691774", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "read_error", - "size_mismatch_oi", - "obj_size_oi_mismatch" + "size_mismatch_info", + "obj_size_info_mismatch" ], "errors": [ "size_mismatch" @@ -2461,7 +4154,41 @@ EOF "primary": false } ], - "selected_object_info": "3:9babd184:::EOBJ2:head(29'2 client.4217.0:1 dirty|data_digest|omap_digest s 7 uv 2 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ2", + "key": "", + "snapid": -2, + "hash": 562812377, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "33'2", + "prior_version": "0'0", + "last_reqid": "client.4224.0:1", + "user_version": 2, + "size": 7, + "mtime": "2018-04-05 14:30:14.152945", + "local_mtime": "2018-04-05 14:30:14.154014", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "ec_hash_error" ], @@ -2503,7 +4230,41 @@ EOF "primary": false } ], - "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ3", + "key": "", + "snapid": -2, + "hash": 3125668237, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "39'3", + "prior_version": "0'0", + "last_reqid": "client.4258.0:1", + "user_version": 3, + "size": 7, + "mtime": "2018-04-05 14:30:18.875544", + "local_mtime": "2018-04-05 14:30:18.880153", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ "missing" ], @@ -2520,30 +4281,15 @@ EOF "shards": [ { "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "bad-val", - "name": "_key1-EOBJ4" + "name": "key1-EOBJ4" }, { "Base64": false, "value": "val2-EOBJ4", - "name": "_key2-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key2-EOBJ4" } ], "data_digest": "0x04cfa72f", @@ -2563,30 +4309,15 @@ EOF "omap_digest": "0xffffffff", "data_digest": "0x04cfa72f", "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "val1-EOBJ4", - "name": "_key1-EOBJ4" + "name": "key1-EOBJ4" }, { "Base64": false, "value": "val2-EOBJ4", - "name": "_key2-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key2-EOBJ4" } ] }, @@ -2599,35 +4330,54 @@ EOF "omap_digest": "0xffffffff", "data_digest": "0x04cfa72f", "attrs": [ - { - "Base64": true, - "value": "", - "name": "_" - }, { "Base64": false, "value": "val1-EOBJ4", - "name": "_key1-EOBJ4" + "name": "key1-EOBJ4" }, { "Base64": false, "value": "val3-EOBJ4", - "name": "_key3-EOBJ4" - }, - { - "Base64": true, - "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E", - "name": "hinfo_key" - }, - { - "Base64": true, - "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "name": "snapset" + "name": "key3-EOBJ4" } ] } ], - "selected_object_info": "3:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ4", + "key": "", + "snapid": -2, + "hash": 1618759290, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "45'6", + "prior_version": "45'5", + "last_reqid": "client.4296.0:1", + "user_version": 6, + "size": 7, + "mtime": "2018-04-05 14:30:22.271983", + "local_mtime": "2018-04-05 14:30:22.272840", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [], "errors": [ "attr_value_mismatch", @@ -2653,12 +4403,47 @@ EOF "primary": false }, { + "object_info": { + "oid": { + "oid": "EOBJ5", + "key": "", + "snapid": -2, + "hash": 2918945441, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "59'7", + "prior_version": "0'0", + "last_reqid": "client.4384.0:1", + "user_version": 7, + "size": 7, + "mtime": "2018-04-05 14:30:35.162395", + "local_mtime": "2018-04-05 14:30:35.166390", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "size": 4096, "shard": 0, "errors": [ - "size_mismatch_oi", + "size_mismatch_info", "ec_size_error", - "obj_size_oi_mismatch" + "obj_size_info_mismatch" ], "osd": 1, "primary": true @@ -2673,11 +4458,45 @@ EOF "primary": false } ], - "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "EOBJ5", + "key": "", + "snapid": -2, + "hash": 2918945441, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "59'7", + "prior_version": "0'0", + "last_reqid": "client.4384.0:1", + "user_version": 7, + "size": 7, + "mtime": "2018-04-05 14:30:35.162395", + "local_mtime": "2018-04-05 14:30:35.166390", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "union_shard_errors": [ - "size_mismatch_oi", + "size_mismatch_info", "ec_size_error", - "obj_size_oi_mismatch" + "obj_size_info_mismatch" ], "errors": [ "size_mismatch" @@ -2689,6 +4508,235 @@ EOF "nspace": "", "name": "EOBJ5" } + }, + { + "object": { + "name": "EOBJ6", + "nspace": "", + "locator": "", + "snap": "head", + "version": 8 + }, + "errors": [], + "union_shard_errors": [ + "read_error", + "hinfo_missing", + "hinfo_corrupted" + ], + "selected_object_info": { + "oid": { + "oid": "EOBJ6", + "key": "", + "snapid": -2, + "hash": 3050890866, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "65'8", + "prior_version": "0'0", + "last_reqid": "client.4420.0:1", + "user_version": 8, + "size": 7, + "mtime": "2018-04-05 14:30:40.914673", + "local_mtime": "2018-04-05 14:30:40.917705", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x2ddbf8f5", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, + "shards": [ + { + "osd": 0, + "primary": false, + "shard": 2, + "errors": [ + "read_error", + "hinfo_missing" + ], + "size": 2048 + }, + { + "osd": 1, + "primary": true, + "shard": 0, + "errors": [ + "read_error", + "hinfo_corrupted" + ], + "size": 2048, + "hashinfo": "bad-val" + }, + { + "osd": 2, + "primary": false, + "shard": 1, + "errors": [], + "size": 2048, + "omap_digest": "0xffffffff", + "data_digest": "0x04cfa72f", + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 80717615, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 80717615, + "shard": 2 + } + ], + "total_chunk_size": 2048 + } + } + ] + }, + { + "object": { + "name": "EOBJ7", + "nspace": "", + "locator": "", + "snap": "head", + "version": 10 + }, + "errors": [ + "hinfo_inconsistency" + ], + "union_shard_errors": [ + "ec_hash_error" + ], + "selected_object_info": { + "oid": { + "oid": "EOBJ7", + "key": "", + "snapid": -2, + "hash": 3258066308, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "75'10", + "prior_version": "75'9", + "last_reqid": "client.4486.0:1", + "user_version": 10, + "size": 34, + "mtime": "2018-04-05 14:30:50.995009", + "local_mtime": "2018-04-05 14:30:50.996112", + "lost": 0, + "flags": [ + "dirty", + "data_digest", + "omap_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x136e4e27", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, + "shards": [ + { + "osd": 0, + "primary": false, + "shard": 2, + "errors": [ + "ec_hash_error" + ], + "size": 2048, + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 80717615, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 80717615, + "shard": 2 + } + ], + "total_chunk_size": 2048 + } + }, + { + "osd": 1, + "primary": true, + "shard": 0, + "errors": [], + "size": 2048, + "omap_digest": "0xffffffff", + "data_digest": "0x5b7455a8", + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 1534350760, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 1534350760, + "shard": 2 + } + ], + "total_chunk_size": 2048 + } + }, + { + "osd": 2, + "primary": false, + "shard": 1, + "errors": [], + "size": 2048, + "omap_digest": "0xffffffff", + "data_digest": "0x5b7455a8", + "hashinfo": { + "cumulative_shard_hashes": [ + { + "hash": 1534350760, + "shard": 0 + }, + { + "hash": 1534491824, + "shard": 1 + }, + { + "hash": 1534350760, + "shard": 2 + } + ], + "total_chunk_size": 2048 + } + } + ] } ], "epoch": 0 @@ -2697,7 +4745,7 @@ EOF fi - jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson + jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 if test $getjson = "yes" then @@ -2710,7 +4758,7 @@ EOF jq '.' $dir/json > save${num}.json fi - if which jsonschema > /dev/null; + if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null; then jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 fi @@ -2881,7 +4929,7 @@ function TEST_corrupt_snapset_scrub_rep() { rados list-inconsistent-obj $pg > $dir/json || return 1 - jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson + jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson { "epoch": 34, "inconsistents": [ @@ -2897,21 +4945,80 @@ function TEST_corrupt_snapset_scrub_rep() { "snapset_inconsistency" ], "union_shard_errors": [], - "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(27'8 client.4143.0:1 dirty|omap|data_digest s 21 uv 8 dd 53acb008 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ1", + "key": "", + "snapid": -2, + "hash": 1454963827, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "24'8", + "prior_version": "21'3", + "last_reqid": "client.4195.0:1", + "user_version": 8, + "size": 21, + "mtime": "2018-04-05 14:35:43.286117", + "local_mtime": "2018-04-05 14:35:43.288990", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x53acb008", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "shards": [ { "osd": 0, "primary": false, "errors": [], "size": 21, - "snapset": "1=[1]:{1=[1]}" + "snapset": { + "clones": [ + { + "overlap": "[]", + "size": 7, + "snap": 1, + "snaps": [ + 1 + ] + } + ], + "head_exists": 1, + "snap_context": { + "seq": 1, + "snaps": [ + 1 + ] + } + } }, { "osd": 1, "primary": true, "errors": [], "size": 21, - "snapset": "0=[]:[]+stray_clone_snaps={1=[1]}" + "snapset": { + "clones": [], + "head_exists": 0, + "snap_context": { + "seq": 0, + "snaps": [] + } + } } ] }, @@ -2927,21 +5034,80 @@ function TEST_corrupt_snapset_scrub_rep() { "snapset_inconsistency" ], "union_shard_errors": [], - "selected_object_info": "3:e97ce31e:::ROBJ2:head(31'10 client.4155.0:1 dirty|omap|data_digest s 21 uv 10 dd 53acb008 alloc_hint [0 0 0])", + "selected_object_info": { + "oid": { + "oid": "ROBJ2", + "key": "", + "snapid": -2, + "hash": 2026323607, + "max": 0, + "pool": 3, + "namespace": "" + }, + "version": "28'10", + "prior_version": "23'6", + "last_reqid": "client.4223.0:1", + "user_version": 10, + "size": 21, + "mtime": "2018-04-05 14:35:48.326856", + "local_mtime": "2018-04-05 14:35:48.328097", + "lost": 0, + "flags": [ + "dirty", + "omap", + "data_digest" + ], + "truncate_seq": 0, + "truncate_size": 0, + "data_digest": "0x53acb008", + "omap_digest": "0xffffffff", + "expected_object_size": 0, + "expected_write_size": 0, + "alloc_hint_flags": 0, + "manifest": { + "type": 0 + }, + "watchers": {} + }, "shards": [ { "osd": 0, "primary": false, "errors": [], "size": 21, - "snapset": "0=[]:[]+stray_clone_snaps={1=[1]}" + "snapset": { + "clones": [], + "head_exists": 0, + "snap_context": { + "seq": 0, + "snaps": [] + } + } }, { "osd": 1, "primary": true, "errors": [], "size": 21, - "snapset": "1=[1]:{1=[1]}" + "snapset": { + "clones": [ + { + "overlap": "[]", + "size": 7, + "snap": 1, + "snaps": [ + 1 + ] + } + ], + "head_exists": 1, + "snap_context": { + "seq": 1, + "snaps": [ + 1 + ] + } + } } ] } @@ -2949,14 +5115,14 @@ function TEST_corrupt_snapset_scrub_rep() { } EOF - jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson + jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 if test $getjson = "yes" then jq '.' $dir/json > save6.json fi - if which jsonschema > /dev/null; + if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null; then jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 fi diff --git a/ceph/qa/standalone/scrub/osd-scrub-snaps.sh b/ceph/qa/standalone/scrub/osd-scrub-snaps.sh index 4c03bdb9e..cf6f67b6d 100755 --- a/ceph/qa/standalone/scrub/osd-scrub-snaps.sh +++ b/ceph/qa/standalone/scrub/osd-scrub-snaps.sh @@ -16,6 +16,10 @@ # source $CEPH_ROOT/qa/standalone/ceph-helpers.sh +# Test development and debugging +# Set to "yes" in order to ignore diff errors and save results to update test +getjson="no" + function run() { local dir=$1 shift @@ -27,23 +31,26 @@ function run() { local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} for func in $funcs ; do + setup $dir || return 1 $func $dir || return 1 + teardown $dir || return 1 done } function TEST_scrub_snaps() { local dir=$1 local poolname=test + local OBJS=15 + local OSDS=1 TESTDATA="testdata.$$" - setup $dir || return 1 - run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mon $dir a --osd_pool_default_size=$OSDS || return 1 run_mgr $dir x || return 1 - run_osd $dir 0 || return 1 - - create_rbd_pool || return 1 - wait_for_clean || return 1 + for osd in $(seq 0 $(expr $OSDS - 1)) + do + run_osd $dir $osd || return 1 + done # Create a pool with a single pg create_pool $poolname 1 1 @@ -51,11 +58,12 @@ function TEST_scrub_snaps() { poolid=$(ceph osd dump | grep "^pool.*[']test[']" | awk '{ print $2 }') dd if=/dev/urandom of=$TESTDATA bs=1032 count=1 - for i in `seq 1 15` + for i in `seq 1 $OBJS` do rados -p $poolname put obj${i} $TESTDATA done + local primary=$(get_primary $poolname obj1) SNAP=1 rados -p $poolname mksnap snap${SNAP} dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP} @@ -97,66 +105,67 @@ function TEST_scrub_snaps() { kill_daemons $dir TERM osd || return 1 - # Don't need to ceph_objectstore_tool function because osd stopped + # Don't need to use ceph_objectstore_tool() function because osd stopped - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj1)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" --force remove + JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj1)" + ceph-objectstore-tool --data-path $dir/${primary} "$JSON" --force remove - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --op list obj5 | grep \"snapid\":2)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" remove + JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj5 | grep \"snapid\":2)" + ceph-objectstore-tool --data-path $dir/${primary} "$JSON" remove - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --op list obj5 | grep \"snapid\":1)" + JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj5 | grep \"snapid\":1)" OBJ5SAVE="$JSON" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" remove + ceph-objectstore-tool --data-path $dir/${primary} "$JSON" remove - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --op list obj5 | grep \"snapid\":4)" + JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj5 | grep \"snapid\":4)" dd if=/dev/urandom of=$TESTDATA bs=256 count=18 - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" set-bytes $TESTDATA + ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-bytes $TESTDATA - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj3)" + JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj3)" dd if=/dev/urandom of=$TESTDATA bs=256 count=15 - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" set-bytes $TESTDATA + ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-bytes $TESTDATA - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --op list obj4 | grep \"snapid\":7)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" remove + JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj4 | grep \"snapid\":7)" + ceph-objectstore-tool --data-path $dir/${primary} "$JSON" remove - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj2)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" rm-attr snapset + JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj2)" + ceph-objectstore-tool --data-path $dir/${primary} "$JSON" rm-attr snapset # Create a clone which isn't in snapset and doesn't have object info JSON="$(echo "$OBJ5SAVE" | sed s/snapid\":1/snapid\":7/)" dd if=/dev/urandom of=$TESTDATA bs=256 count=7 - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" set-bytes $TESTDATA + ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-bytes $TESTDATA rm -f $TESTDATA - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj6)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj7)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset corrupt - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj8)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset seq - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj9)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset clone_size - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj10)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset clone_overlap - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj11)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset clones - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj12)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset head - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj13)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset snaps - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj14)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset size + JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj6)" + ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset + JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj7)" + ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset corrupt + JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj8)" + ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset seq + JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj9)" + ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset clone_size + JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj10)" + ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset clone_overlap + JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj11)" + ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset clones + JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj12)" + ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset head + JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj13)" + ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset snaps + JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj14)" + ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset size echo "garbage" > $dir/bad - JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj15)" - ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" set-attr snapset $dir/bad + JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj15)" + ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-attr snapset $dir/bad rm -f $dir/bad - run_osd $dir 0 || return 1 - create_rbd_pool || return 1 - wait_for_clean || return 1 + for osd in $(seq 0 $(expr $OSDS - 1)) + do + run_osd $dir $osd || return 1 + done local pgid="${poolid}.0" if ! pg_scrub "$pgid" ; then @@ -172,7 +181,6 @@ function TEST_scrub_snaps() { test $(jq -r '.[0]' $dir/json) = $pgid || return 1 rados list-inconsistent-snapset $pgid > $dir/json || return 1 - test $(jq '.inconsistents | length' $dir/json) = "21" || return 1 local jqfilter='.inconsistents' local sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)' @@ -272,7 +280,7 @@ function TEST_scrub_snaps() { }, { "errors": [ - "oi_attr_missing", + "info_missing", "headless" ], "snap": 7, @@ -280,6 +288,32 @@ function TEST_scrub_snaps() { "nspace": "", "name": "obj5" }, + { + "name": "obj10", + "nspace": "", + "locator": "", + "snap": "head", + "snapset": { + "head_exists": 1, + "snap_context": { + "seq": 1, + "snaps": [ + 1 + ] + }, + "clones": [ + { + "snap": 1, + "size": 1032, + "overlap": "????", + "snaps": [ + 1 + ] + } + ] + }, + "errors": [] + }, { "extra clones": [ 1 @@ -290,7 +324,17 @@ function TEST_scrub_snaps() { "snap": "head", "locator": "", "nspace": "", - "name": "obj11" + "name": "obj11", + "snapset": { + "head_exists": 1, + "snap_context": { + "seq": 1, + "snaps": [ + 1 + ] + }, + "clones": [] + } }, { "errors": [ @@ -299,11 +343,56 @@ function TEST_scrub_snaps() { "snap": "head", "locator": "", "nspace": "", - "name": "obj12" + "name": "obj12", + "snapset": { + "head_exists": 0, + "snap_context": { + "seq": 1, + "snaps": [ + 1 + ] + }, + "clones": [ + { + "snap": 1, + "size": 1032, + "overlap": "[]", + "snaps": [ + 1 + ] + } + ] + } + }, + { + "name": "obj14", + "nspace": "", + "locator": "", + "snap": "head", + "snapset": { + "head_exists": 1, + "snap_context": { + "seq": 1, + "snaps": [ + 1 + ] + }, + "clones": [ + { + "snap": 1, + "size": 1033, + "overlap": "[]", + "snaps": [ + 1 + ] + } + ] + }, + "errors": [] }, { "errors": [ - "ss_attr_corrupted" + "snapset_corrupted" ], "snap": "head", "locator": "", @@ -316,7 +405,7 @@ function TEST_scrub_snaps() { 4 ], "errors": [ - "ss_attr_missing", + "snapset_missing", "extra_clones" ], "snap": "head", @@ -331,7 +420,37 @@ function TEST_scrub_snaps() { "snap": "head", "locator": "", "nspace": "", - "name": "obj3" + "name": "obj3", + "snapset": { + "head_exists": 1, + "snap_context": { + "seq": 3, + "snaps": [ + 3, + 2, + 1 + ] + }, + "clones": [ + { + "snap": 1, + "size": 1032, + "overlap": "[]", + "snaps": [ + 1 + ] + }, + { + "snap": 3, + "size": 256, + "overlap": "[]", + "snaps": [ + 3, + 2 + ] + } + ] + } }, { "missing": [ @@ -343,7 +462,38 @@ function TEST_scrub_snaps() { "snap": "head", "locator": "", "nspace": "", - "name": "obj4" + "name": "obj4", + "snapset": { + "head_exists": 1, + "snap_context": { + "seq": 7, + "snaps": [ + 7, + 6, + 5, + 4, + 3, + 2, + 1 + ] + }, + "clones": [ + { + "snap": 7, + "size": 1032, + "overlap": "[]", + "snaps": [ + 7, + 6, + 5, + 4, + 3, + 2, + 1 + ] + } + ] + } }, { "missing": [ @@ -360,7 +510,57 @@ function TEST_scrub_snaps() { "snap": "head", "locator": "", "nspace": "", - "name": "obj5" + "name": "obj5", + "snapset": { + "head_exists": 1, + "snap_context": { + "seq": 6, + "snaps": [ + 6, + 5, + 4, + 3, + 2, + 1 + ] + }, + "clones": [ + { + "snap": 1, + "size": 1032, + "overlap": "[]", + "snaps": [ + 1 + ] + }, + { + "snap": 2, + "size": 256, + "overlap": "[]", + "snaps": [ + 2 + ] + }, + { + "snap": 4, + "size": 512, + "overlap": "[]", + "snaps": [ + 4, + 3 + ] + }, + { + "snap": 6, + "size": 1024, + "overlap": "[]", + "snaps": [ + 6, + 5 + ] + } + ] + } }, { "extra clones": [ @@ -372,7 +572,17 @@ function TEST_scrub_snaps() { "snap": "head", "locator": "", "nspace": "", - "name": "obj6" + "name": "obj6", + "snapset": { + "head_exists": 1, + "snap_context": { + "seq": 1, + "snaps": [ + 1 + ] + }, + "clones": [] + } }, { "extra clones": [ @@ -385,16 +595,69 @@ function TEST_scrub_snaps() { "snap": "head", "locator": "", "nspace": "", - "name": "obj7" + "name": "obj7", + "snapset": { + "head_exists": 0, + "snap_context": { + "seq": 0, + "snaps": [] + }, + "clones": [] + } }, { "errors": [ - "snapset_mismatch" + "snapset_error" ], "snap": "head", "locator": "", "nspace": "", - "name": "obj8" + "name": "obj8", + "snapset": { + "head_exists": 1, + "snap_context": { + "seq": 0, + "snaps": [ + 1 + ] + }, + "clones": [ + { + "snap": 1, + "size": 1032, + "overlap": "[]", + "snaps": [ + 1 + ] + } + ] + } + }, + { + "name": "obj9", + "nspace": "", + "locator": "", + "snap": "head", + "snapset": { + "head_exists": 1, + "snap_context": { + "seq": 1, + "snaps": [ + 1 + ] + }, + "clones": [ + { + "snap": 1, + "size": "????", + "overlap": "[]", + "snaps": [ + 1 + ] + } + ] + }, + "errors": [] } ], "epoch": 20 @@ -402,9 +665,13 @@ function TEST_scrub_snaps() { EOF jq "$jqfilter" $dir/json | python -c "$sortkeys" > $dir/csjson - diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || return 1 + diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 + if test $getjson = "yes" + then + jq '.' $dir/json > save1.json + fi - if which jsonschema > /dev/null; + if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null; then jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-snap.json || return 1 fi @@ -455,15 +722,13 @@ EOF for err_string in "${err_strings[@]}" do - if ! grep "$err_string" $dir/osd.0.log > /dev/null; + if ! grep "$err_string" $dir/osd.${primary}.log > /dev/null; then echo "Missing log message '$err_string'" ERRORS=$(expr $ERRORS + 1) fi done - teardown $dir || return 1 - if [ $ERRORS != "0" ]; then echo "TEST FAILED WITH $ERRORS ERRORS" diff --git a/ceph/qa/standalone/scrub/osd-scrub-test.sh b/ceph/qa/standalone/scrub/osd-scrub-test.sh new file mode 100755 index 000000000..e6c93684a --- /dev/null +++ b/ceph/qa/standalone/scrub/osd-scrub-test.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2018 Red Hat +# +# Author: David Zafman +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7138" # git grep '\<7138\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + $func $dir || return 1 + done +} + +function TEST_scrub_test() { + local dir=$1 + local poolname=test + local OSDS=3 + local objects=15 + + TESTDATA="testdata.$$" + + setup $dir || return 1 + run_mon $dir a --osd_pool_default_size=3 || return 1 + run_mgr $dir x || return 1 + for osd in $(seq 0 $(expr $OSDS - 1)) + do + run_osd $dir $osd || return 1 + done + + # Create a pool with a single pg + create_pool $poolname 1 1 + wait_for_clean || return 1 + poolid=$(ceph osd dump | grep "^pool.*[']${poolname}[']" | awk '{ print $2 }') + + dd if=/dev/urandom of=$TESTDATA bs=1032 count=1 + for i in `seq 1 $objects` + do + rados -p $poolname put obj${i} $TESTDATA + done + rm -f $TESTDATA + + local primary=$(get_primary $poolname obj1) + local otherosd=$(get_not_primary $poolname obj1) + if [ "$otherosd" = "2" ]; + then + local anotherosd="0" + else + local anotherosd="2" + fi + + objectstore_tool $dir $anotherosd obj1 set-bytes /etc/fstab + + local pgid="${poolid}.0" + pg_deep_scrub "$pgid" || return 1 + + ceph pg dump pgs | grep ^${pgid} | grep -q -- +inconsistent || return 1 + test "$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_scrub_errors')" = "2" || return 1 + + ceph osd out $primary + wait_for_clean || return 1 + + pg_deep_scrub "$pgid" || return 1 + + test "$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_scrub_errors')" = "2" || return 1 + test "$(ceph pg $pgid query | jq '.peer_info[0].stats.stat_sum.num_scrub_errors')" = "2" || return 1 + ceph pg dump pgs | grep ^${pgid} | grep -q -- +inconsistent || return 1 + + ceph osd in $primary + wait_for_clean || return 1 + + repair "$pgid" || return 1 + wait_for_clean || return 1 + + # This sets up the test after we've repaired with previous primary has old value + test "$(ceph pg $pgid query | jq '.peer_info[0].stats.stat_sum.num_scrub_errors')" = "2" || return 1 + ceph pg dump pgs | grep ^${pgid} | grep -vq -- +inconsistent || return 1 + + ceph osd out $primary + wait_for_clean || return 1 + + test "$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_scrub_errors')" = "0" || return 1 + test "$(ceph pg $pgid query | jq '.peer_info[0].stats.stat_sum.num_scrub_errors')" = "0" || return 1 + test "$(ceph pg $pgid query | jq '.peer_info[1].stats.stat_sum.num_scrub_errors')" = "0" || return 1 + ceph pg dump pgs | grep ^${pgid} | grep -vq -- +inconsistent || return 1 + + teardown $dir || return 1 +} + +main osd-scrub-test "$@" + +# Local Variables: +# compile-command: "cd ../.. ; make -j4 && \ +# qa/standalone/scrub/osd-scrub-test.sh" diff --git a/ceph/qa/standalone/special/ceph_objectstore_tool.py b/ceph/qa/standalone/special/ceph_objectstore_tool.py index 63333bef2..0c6097c1f 100755 --- a/ceph/qa/standalone/special/ceph_objectstore_tool.py +++ b/ceph/qa/standalone/special/ceph_objectstore_tool.py @@ -1027,7 +1027,7 @@ def main(argv): # Specify a bad --op command cmd = (CFSD_PREFIX + "--op oops").format(osd=ONEOSD) - ERRORS += test_failure(cmd, "Must provide --op (info, log, remove, mkfs, fsck, repair, export, export-remove, import, list, fix-lost, list-pgs, rm-past-intervals, dump-journal, dump-super, meta-list, get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete, dump-import)") + ERRORS += test_failure(cmd, "Must provide --op (info, log, remove, mkfs, fsck, repair, export, export-remove, import, list, fix-lost, list-pgs, rm-past-intervals, dump-journal, dump-super, meta-list, get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete, dump-import, trim-pg-log)") # Provide just the object param not a command cmd = (CFSD_PREFIX + "object").format(osd=ONEOSD) diff --git a/ceph/qa/suites/fs/basic_functional/tasks/cephfs_scrub_tests.yaml b/ceph/qa/suites/fs/basic_functional/tasks/cephfs_scrub_tests.yaml index 30b3a96e2..f79784c0b 100644 --- a/ceph/qa/suites/fs/basic_functional/tasks/cephfs_scrub_tests.yaml +++ b/ceph/qa/suites/fs/basic_functional/tasks/cephfs_scrub_tests.yaml @@ -4,6 +4,7 @@ overrides: - Scrub error on inode - Behind on trimming - Metadata damage detected + - bad backtrace on inode - overall HEALTH_ - (MDS_TRIM) conf: @@ -14,3 +15,4 @@ tasks: - cephfs_test_runner: modules: - tasks.cephfs.test_scrub_checks + - tasks.cephfs.test_scrub diff --git a/ceph/qa/suites/fs/basic_functional/tasks/mds-full.yaml b/ceph/qa/suites/fs/basic_functional/tasks/mds-full.yaml index 35c69c790..eab120590 100644 --- a/ceph/qa/suites/fs/basic_functional/tasks/mds-full.yaml +++ b/ceph/qa/suites/fs/basic_functional/tasks/mds-full.yaml @@ -6,9 +6,12 @@ overrides: log-whitelist: - OSD full dropping all updates - OSD near full + - pausewr flag - failsafe engaged, dropping updates - failsafe disengaged, no longer dropping - is full \(reached quota + - POOL_FULL + - POOL_BACKFILLFULL conf: mon: mon osd nearfull ratio: 0.6 diff --git a/ceph/qa/suites/kcephfs/cephfs/clusters/1-mds-1-client.yaml b/ceph/qa/suites/kcephfs/cephfs/clusters/1-mds-1-client.yaml new file mode 120000 index 000000000..260d3e904 --- /dev/null +++ b/ceph/qa/suites/kcephfs/cephfs/clusters/1-mds-1-client.yaml @@ -0,0 +1 @@ +../../../../cephfs/clusters/1-mds-1-client.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/cephfs/clusters/fixed-3-cephfs.yaml b/ceph/qa/suites/kcephfs/cephfs/clusters/fixed-3-cephfs.yaml deleted file mode 120000 index a482e6504..000000000 --- a/ceph/qa/suites/kcephfs/cephfs/clusters/fixed-3-cephfs.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../../clusters/fixed-3-cephfs.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/jewel-x/point-to-point-x/% b/ceph/qa/suites/kcephfs/cephfs/overrides/+ similarity index 100% rename from ceph/qa/suites/upgrade/jewel-x/point-to-point-x/% rename to ceph/qa/suites/kcephfs/cephfs/overrides/+ diff --git a/ceph/qa/suites/kcephfs/cephfs/overrides/debug.yaml b/ceph/qa/suites/kcephfs/cephfs/overrides/debug.yaml new file mode 120000 index 000000000..9bc8eb1e7 --- /dev/null +++ b/ceph/qa/suites/kcephfs/cephfs/overrides/debug.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/debug.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/cephfs/overrides/frag_enable.yaml b/ceph/qa/suites/kcephfs/cephfs/overrides/frag_enable.yaml new file mode 120000 index 000000000..e9b2d64d0 --- /dev/null +++ b/ceph/qa/suites/kcephfs/cephfs/overrides/frag_enable.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/frag_enable.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/cephfs/overrides/log-config.yaml b/ceph/qa/suites/kcephfs/cephfs/overrides/log-config.yaml new file mode 120000 index 000000000..a8c4d2190 --- /dev/null +++ b/ceph/qa/suites/kcephfs/cephfs/overrides/log-config.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/log-config.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/cephfs/overrides/osd-asserts.yaml b/ceph/qa/suites/kcephfs/cephfs/overrides/osd-asserts.yaml new file mode 120000 index 000000000..6b07e51a9 --- /dev/null +++ b/ceph/qa/suites/kcephfs/cephfs/overrides/osd-asserts.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/osd-asserts.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/cephfs/overrides/whitelist_health.yaml b/ceph/qa/suites/kcephfs/cephfs/overrides/whitelist_health.yaml new file mode 120000 index 000000000..440e747fb --- /dev/null +++ b/ceph/qa/suites/kcephfs/cephfs/overrides/whitelist_health.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/whitelist_health.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/cephfs/overrides/whitelist_wrongly_marked_down.yaml b/ceph/qa/suites/kcephfs/cephfs/overrides/whitelist_wrongly_marked_down.yaml new file mode 120000 index 000000000..a26a6579c --- /dev/null +++ b/ceph/qa/suites/kcephfs/cephfs/overrides/whitelist_wrongly_marked_down.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/whitelist_wrongly_marked_down.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/mixed-clients/clusters/1-mds-2-client.yaml b/ceph/qa/suites/kcephfs/mixed-clients/clusters/1-mds-2-client.yaml new file mode 120000 index 000000000..fb642244a --- /dev/null +++ b/ceph/qa/suites/kcephfs/mixed-clients/clusters/1-mds-2-client.yaml @@ -0,0 +1 @@ +../../../../cephfs/clusters/1-mds-2-client.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/mixed-clients/clusters/2-clients.yaml b/ceph/qa/suites/kcephfs/mixed-clients/clusters/2-clients.yaml deleted file mode 100644 index 90b6cf626..000000000 --- a/ceph/qa/suites/kcephfs/mixed-clients/clusters/2-clients.yaml +++ /dev/null @@ -1,9 +0,0 @@ -roles: -- [mon.a, mgr.x, mds.a, osd.0, osd.1] -- [mon.b, mon.c, osd.2, osd.3] -- [client.0] -- [client.1] -openstack: -- volumes: # attached to each instance - count: 2 - size: 10 # GB diff --git a/ceph/qa/suites/kcephfs/mixed-clients/overrides/+ b/ceph/qa/suites/kcephfs/mixed-clients/overrides/+ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/kcephfs/mixed-clients/overrides/debug.yaml b/ceph/qa/suites/kcephfs/mixed-clients/overrides/debug.yaml new file mode 120000 index 000000000..9bc8eb1e7 --- /dev/null +++ b/ceph/qa/suites/kcephfs/mixed-clients/overrides/debug.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/debug.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/mixed-clients/overrides/frag_enable.yaml b/ceph/qa/suites/kcephfs/mixed-clients/overrides/frag_enable.yaml new file mode 120000 index 000000000..e9b2d64d0 --- /dev/null +++ b/ceph/qa/suites/kcephfs/mixed-clients/overrides/frag_enable.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/frag_enable.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/mixed-clients/overrides/log-config.yaml b/ceph/qa/suites/kcephfs/mixed-clients/overrides/log-config.yaml new file mode 120000 index 000000000..a8c4d2190 --- /dev/null +++ b/ceph/qa/suites/kcephfs/mixed-clients/overrides/log-config.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/log-config.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/mixed-clients/overrides/osd-asserts.yaml b/ceph/qa/suites/kcephfs/mixed-clients/overrides/osd-asserts.yaml new file mode 120000 index 000000000..6b07e51a9 --- /dev/null +++ b/ceph/qa/suites/kcephfs/mixed-clients/overrides/osd-asserts.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/osd-asserts.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/mixed-clients/overrides/whitelist_health.yaml b/ceph/qa/suites/kcephfs/mixed-clients/overrides/whitelist_health.yaml new file mode 120000 index 000000000..440e747fb --- /dev/null +++ b/ceph/qa/suites/kcephfs/mixed-clients/overrides/whitelist_health.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/whitelist_health.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/mixed-clients/overrides/whitelist_wrongly_marked_down.yaml b/ceph/qa/suites/kcephfs/mixed-clients/overrides/whitelist_wrongly_marked_down.yaml new file mode 120000 index 000000000..a26a6579c --- /dev/null +++ b/ceph/qa/suites/kcephfs/mixed-clients/overrides/whitelist_wrongly_marked_down.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/whitelist_wrongly_marked_down.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/recovery/clusters/1-mds-4-client.yaml b/ceph/qa/suites/kcephfs/recovery/clusters/1-mds-4-client.yaml new file mode 120000 index 000000000..51fc2b876 --- /dev/null +++ b/ceph/qa/suites/kcephfs/recovery/clusters/1-mds-4-client.yaml @@ -0,0 +1 @@ +../../../../cephfs/clusters/1-mds-4-client.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/recovery/clusters/4-remote-clients.yaml b/ceph/qa/suites/kcephfs/recovery/clusters/4-remote-clients.yaml deleted file mode 100644 index b1072e3be..000000000 --- a/ceph/qa/suites/kcephfs/recovery/clusters/4-remote-clients.yaml +++ /dev/null @@ -1,12 +0,0 @@ -roles: -- [mon.a, osd.0, osd.1, osd.2, osd.3, mds.a, mds.c, client.2] -- [mgr.x, osd.4, osd.5, osd.6, osd.7, mds.b, mds.d, client.3] -- [client.0] -- [client.1] -openstack: -- volumes: # attached to each instance - count: 2 - size: 10 # GB -log-rotate: - ceph-mds: 10G - ceph-osd: 10G diff --git a/ceph/qa/suites/kcephfs/recovery/overrides/+ b/ceph/qa/suites/kcephfs/recovery/overrides/+ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/kcephfs/recovery/overrides/debug.yaml b/ceph/qa/suites/kcephfs/recovery/overrides/debug.yaml new file mode 120000 index 000000000..9bc8eb1e7 --- /dev/null +++ b/ceph/qa/suites/kcephfs/recovery/overrides/debug.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/debug.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/recovery/overrides/frag_enable.yaml b/ceph/qa/suites/kcephfs/recovery/overrides/frag_enable.yaml new file mode 120000 index 000000000..e9b2d64d0 --- /dev/null +++ b/ceph/qa/suites/kcephfs/recovery/overrides/frag_enable.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/frag_enable.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/recovery/overrides/log-config.yaml b/ceph/qa/suites/kcephfs/recovery/overrides/log-config.yaml new file mode 120000 index 000000000..a8c4d2190 --- /dev/null +++ b/ceph/qa/suites/kcephfs/recovery/overrides/log-config.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/log-config.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/recovery/overrides/osd-asserts.yaml b/ceph/qa/suites/kcephfs/recovery/overrides/osd-asserts.yaml new file mode 120000 index 000000000..6b07e51a9 --- /dev/null +++ b/ceph/qa/suites/kcephfs/recovery/overrides/osd-asserts.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/osd-asserts.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/recovery/overrides/whitelist_health.yaml b/ceph/qa/suites/kcephfs/recovery/overrides/whitelist_health.yaml new file mode 120000 index 000000000..440e747fb --- /dev/null +++ b/ceph/qa/suites/kcephfs/recovery/overrides/whitelist_health.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/whitelist_health.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/recovery/overrides/whitelist_wrongly_marked_down.yaml b/ceph/qa/suites/kcephfs/recovery/overrides/whitelist_wrongly_marked_down.yaml new file mode 120000 index 000000000..a26a6579c --- /dev/null +++ b/ceph/qa/suites/kcephfs/recovery/overrides/whitelist_wrongly_marked_down.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/whitelist_wrongly_marked_down.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/recovery/tasks/mds-full.yaml b/ceph/qa/suites/kcephfs/recovery/tasks/mds-full.yaml index 558a20673..77a0a83e5 100644 --- a/ceph/qa/suites/kcephfs/recovery/tasks/mds-full.yaml +++ b/ceph/qa/suites/kcephfs/recovery/tasks/mds-full.yaml @@ -1,17 +1,27 @@ overrides: ceph: + cephfs_ec_profile: + - disabled log-whitelist: - OSD full dropping all updates - OSD near full + - pausewr flag - failsafe engaged, dropping updates - failsafe disengaged, no longer dropping - is full \(reached quota + - POOL_FULL + - POOL_BACKFILLFULL conf: + mon: + mon osd nearfull ratio: 0.6 + mon osd backfillfull ratio: 0.6 + mon osd full ratio: 0.7 osd: osd mon report interval max: 5 osd objectstore: memstore - memstore device bytes: 100000000 + osd failsafe full ratio: 1.0 + memstore device bytes: 200000000 tasks: - cephfs_test_runner: diff --git a/ceph/qa/suites/kcephfs/thrash/clusters/1-mds-1-client.yaml b/ceph/qa/suites/kcephfs/thrash/clusters/1-mds-1-client.yaml new file mode 120000 index 000000000..260d3e904 --- /dev/null +++ b/ceph/qa/suites/kcephfs/thrash/clusters/1-mds-1-client.yaml @@ -0,0 +1 @@ +../../../../cephfs/clusters/1-mds-1-client.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/thrash/clusters/fixed-3-cephfs.yaml b/ceph/qa/suites/kcephfs/thrash/clusters/fixed-3-cephfs.yaml deleted file mode 120000 index a482e6504..000000000 --- a/ceph/qa/suites/kcephfs/thrash/clusters/fixed-3-cephfs.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../../clusters/fixed-3-cephfs.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/thrash/overrides/+ b/ceph/qa/suites/kcephfs/thrash/overrides/+ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/kcephfs/thrash/overrides/debug.yaml b/ceph/qa/suites/kcephfs/thrash/overrides/debug.yaml new file mode 120000 index 000000000..9bc8eb1e7 --- /dev/null +++ b/ceph/qa/suites/kcephfs/thrash/overrides/debug.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/debug.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/thrash/overrides/frag_enable.yaml b/ceph/qa/suites/kcephfs/thrash/overrides/frag_enable.yaml new file mode 120000 index 000000000..e9b2d64d0 --- /dev/null +++ b/ceph/qa/suites/kcephfs/thrash/overrides/frag_enable.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/frag_enable.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/thrash/overrides/log-config.yaml b/ceph/qa/suites/kcephfs/thrash/overrides/log-config.yaml new file mode 120000 index 000000000..a8c4d2190 --- /dev/null +++ b/ceph/qa/suites/kcephfs/thrash/overrides/log-config.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/log-config.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/thrash/overrides/osd-asserts.yaml b/ceph/qa/suites/kcephfs/thrash/overrides/osd-asserts.yaml new file mode 120000 index 000000000..6b07e51a9 --- /dev/null +++ b/ceph/qa/suites/kcephfs/thrash/overrides/osd-asserts.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/osd-asserts.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/thrash/overrides/whitelist_health.yaml b/ceph/qa/suites/kcephfs/thrash/overrides/whitelist_health.yaml new file mode 120000 index 000000000..440e747fb --- /dev/null +++ b/ceph/qa/suites/kcephfs/thrash/overrides/whitelist_health.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/whitelist_health.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/thrash/overrides/whitelist_wrongly_marked_down.yaml b/ceph/qa/suites/kcephfs/thrash/overrides/whitelist_wrongly_marked_down.yaml new file mode 120000 index 000000000..a26a6579c --- /dev/null +++ b/ceph/qa/suites/kcephfs/thrash/overrides/whitelist_wrongly_marked_down.yaml @@ -0,0 +1 @@ +../../../../cephfs/overrides/whitelist_wrongly_marked_down.yaml \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/thrash/thrashers/mon.yaml b/ceph/qa/suites/kcephfs/thrash/thrashers/mon.yaml index 90612f218..aa876d7d4 100644 --- a/ceph/qa/suites/kcephfs/thrash/thrashers/mon.yaml +++ b/ceph/qa/suites/kcephfs/thrash/thrashers/mon.yaml @@ -1,3 +1,7 @@ +overrides: + ceph: + log-whitelist: + - \(MON_DOWN\) tasks: - install: - ceph: diff --git a/ceph/qa/suites/rados/singleton/all/divergent_priors.yaml b/ceph/qa/suites/rados/singleton/all/divergent_priors.yaml index 604a9e452..00fd13c82 100644 --- a/ceph/qa/suites/rados/singleton/all/divergent_priors.yaml +++ b/ceph/qa/suites/rados/singleton/all/divergent_priors.yaml @@ -17,7 +17,7 @@ overrides: - \(OSDMAP_FLAGS\) - \(OSD_ - \(PG_ - - \(OBJECT_DEGRADED\) + - \(OBJECT_ - \(POOL_APP_NOT_ENABLED\) conf: osd: diff --git a/ceph/qa/suites/rados/singleton/all/divergent_priors2.yaml b/ceph/qa/suites/rados/singleton/all/divergent_priors2.yaml index e2f0245df..57237b5a2 100644 --- a/ceph/qa/suites/rados/singleton/all/divergent_priors2.yaml +++ b/ceph/qa/suites/rados/singleton/all/divergent_priors2.yaml @@ -17,7 +17,7 @@ overrides: - \(OSDMAP_FLAGS\) - \(OSD_ - \(PG_ - - \(OBJECT_DEGRADED\) + - \(OBJECT_ - \(POOL_APP_NOT_ENABLED\) conf: osd: diff --git a/ceph/qa/suites/rbd/maintenance/qemu/xfstests.yaml b/ceph/qa/suites/rbd/maintenance/qemu/xfstests.yaml index ffa012e30..38022f6b9 100644 --- a/ceph/qa/suites/rbd/maintenance/qemu/xfstests.yaml +++ b/ceph/qa/suites/rbd/maintenance/qemu/xfstests.yaml @@ -9,5 +9,6 @@ io_workload: clone: true type: block disks: 3 + time_wait: 120 test: http://git.ceph.com/?p={repo};a=blob_plain;hb={branch};f=qa/run_xfstests_qemu.sh exclude_arch: armv7l diff --git a/ceph/qa/suites/rgw/singleton/overrides.yaml b/ceph/qa/suites/rgw/singleton/overrides.yaml index fc35b09f7..ed4ad591d 100644 --- a/ceph/qa/suites/rgw/singleton/overrides.yaml +++ b/ceph/qa/suites/rgw/singleton/overrides.yaml @@ -4,5 +4,3 @@ overrides: conf: client: debug rgw: 20 - rgw: - frontend: civetweb diff --git a/ceph/qa/suites/rgw/verify/overrides.yaml b/ceph/qa/suites/rgw/verify/overrides.yaml index a9ffd29bb..ed3dec8e2 100644 --- a/ceph/qa/suites/rgw/verify/overrides.yaml +++ b/ceph/qa/suites/rgw/verify/overrides.yaml @@ -6,5 +6,4 @@ overrides: rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= rgw crypt require ssl: false rgw: - frontend: civetweb compression type: random diff --git a/ceph/qa/suites/upgrade/jewel-x/ceph-deploy/jewel-luminous.yaml b/ceph/qa/suites/upgrade/jewel-x/ceph-deploy/jewel-luminous.yaml index 9d44eed30..7332b4afa 100644 --- a/ceph/qa/suites/upgrade/jewel-x/ceph-deploy/jewel-luminous.yaml +++ b/ceph/qa/suites/upgrade/jewel-x/ceph-deploy/jewel-luminous.yaml @@ -26,7 +26,7 @@ openstack: size: 30 # reluctantely :( hard-coded machine type # it will override command line args with teuthology-suite -machine_type: vps +machine_type: ovh roles: - - mon.a - mds.a diff --git a/ceph/qa/suites/upgrade/jewel-x/point-to-point-x/distros/centos_7.3.yaml b/ceph/qa/suites/upgrade/jewel-x/point-to-point-x/distros/centos_7.3.yaml deleted file mode 120000 index c79327bea..000000000 --- a/ceph/qa/suites/upgrade/jewel-x/point-to-point-x/distros/centos_7.3.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../../../distros/all/centos_7.3.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/jewel-x/point-to-point-x/distros/ubuntu_14.04.yaml b/ceph/qa/suites/upgrade/jewel-x/point-to-point-x/distros/ubuntu_14.04.yaml deleted file mode 120000 index 6237042e2..000000000 --- a/ceph/qa/suites/upgrade/jewel-x/point-to-point-x/distros/ubuntu_14.04.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../../../distros/all/ubuntu_14.04.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/jewel-x/point-to-point-x/point-to-point-upgrade.yaml b/ceph/qa/suites/upgrade/jewel-x/point-to-point-x/point-to-point-upgrade.yaml deleted file mode 100644 index d68c258c0..000000000 --- a/ceph/qa/suites/upgrade/jewel-x/point-to-point-x/point-to-point-upgrade.yaml +++ /dev/null @@ -1,236 +0,0 @@ -meta: -- desc: | - Run ceph on two nodes, using one of them as a client, - with a separate client-only node. - Use xfs beneath the osds. - install ceph/jewel v10.2.0 point version - run workload and upgrade-sequence in parallel - install ceph/jewel latest version - run workload and upgrade-sequence in parallel - install ceph/-x version (jewel or kraken) - run workload and upgrade-sequence in parallel -overrides: - ceph: - log-whitelist: - - reached quota - - scrub - - osd_map_max_advance - - wrongly marked - - overall HEALTH_ - - \(MGR_DOWN\) - - \(OSD_ - - \(PG_ - - \(CACHE_ - fs: xfs - conf: - global: - mon warn on pool no app: false - mon: - mon debug unsafe allow tier with nonempty snaps: true - osd: - osd map max advance: 1000 - osd map cache size: 1100 -roles: -- - mon.a - - mds.a - - osd.0 - - osd.1 - - osd.2 - - mgr.x -- - mon.b - - mon.c - - osd.3 - - osd.4 - - osd.5 - - client.0 -- - client.1 -openstack: -- volumes: # attached to each instance - count: 3 - size: 30 # GB -tasks: -- print: "**** v10.2.0 about to install" -- install: - tag: v10.2.0 - exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev', 'librgw2'] -- print: "**** done v10.2.0 install" -- ceph: - fs: xfs - skip_mgr_daemons: true - add_osds_to_crush: true -- print: "**** done ceph xfs" -- sequential: - - workload -- print: "**** done workload v10.2.0" -- install.upgrade: - exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev'] - mon.a: - branch: jewel - mon.b: - branch: jewel - # Note that client.a IS NOT upgraded at this point - #client.1: - #branch: jewel -- parallel: - - workload_jewel - - upgrade-sequence_jewel -- print: "**** done parallel jewel branch" -- install.upgrade: - exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev'] - client.1: - branch: jewel -- print: "**** done branch: jewel install.upgrade on client.1" -- install.upgrade: - mon.a: - mon.b: -- print: "**** done branch: -x install.upgrade on mon.a and mon.b" -- parallel: - - workload_x - - upgrade-sequence_x -- print: "**** done parallel -x branch" -- exec: - osd.0: - - ceph osd set-require-min-compat-client luminous -# Run librados tests on the -x upgraded cluster -- install.upgrade: - client.1: -- workunit: - branch: jewel - clients: - client.1: - - rados/test-upgrade-v11.0.0.sh - - cls -- print: "**** done final test on -x cluster" -####################### -workload: - sequential: - - workunit: - clients: - client.0: - - suites/blogbench.sh -workload_jewel: - full_sequential: - - workunit: - branch: jewel - clients: - client.1: - - rados/test.sh - - cls - env: - CLS_RBD_GTEST_FILTER: '*:-TestClsRbd.mirror_image' - - print: "**** done rados/test.sh & cls workload_jewel" - - sequential: - - rgw: [client.0] - - print: "**** done rgw workload_jewel" - - s3tests: - client.0: - force-branch: ceph-jewel - rgw_server: client.0 - scan_for_encryption_keys: false - - print: "**** done s3tests workload_jewel" -upgrade-sequence_jewel: - sequential: - - print: "**** done branch: jewel install.upgrade" - - ceph.restart: [mds.a] - - sleep: - duration: 60 - - ceph.restart: [osd.0] - - sleep: - duration: 30 - - ceph.restart: [osd.1] - - sleep: - duration: 30 - - ceph.restart: [osd.2] - - sleep: - duration: 30 - - ceph.restart: [osd.3] - - sleep: - duration: 30 - - ceph.restart: [osd.4] - - sleep: - duration: 30 - - ceph.restart: [osd.5] - - sleep: - duration: 60 - - ceph.restart: [mon.a] - - sleep: - duration: 60 - - ceph.restart: [mon.b] - - sleep: - duration: 60 - - ceph.restart: [mon.c] - - sleep: - duration: 60 - - print: "**** done ceph.restart all jewel branch mds/osd/mon" -workload_x: - sequential: - - workunit: - branch: jewel - clients: - client.1: - - rados/test-upgrade-v11.0.0-noec.sh - - cls - env: - CLS_RBD_GTEST_FILTER: '*:-TestClsRbd.mirror_image' - - print: "**** done rados/test-upgrade-v11.0.0.sh & cls workload_x NOT upgraded client" - - workunit: - branch: jewel - clients: - client.0: - - rados/test-upgrade-v11.0.0-noec.sh - - cls - - print: "**** done rados/test-upgrade-v11.0.0.sh & cls workload_x upgraded client" - - rgw: [client.1] - - print: "**** done rgw workload_x" - - s3tests: - client.1: - force-branch: ceph-jewel - rgw_server: client.1 - scan_for_encryption_keys: false - - print: "**** done s3tests workload_x" -upgrade-sequence_x: - sequential: - - ceph.restart: [mds.a] - - sleep: - duration: 60 - - ceph.restart: [mon.a] - - sleep: - duration: 60 - - ceph.restart: [mon.b] - - sleep: - duration: 60 - - ceph.restart: [mon.c] - - sleep: - duration: 60 - - ceph.restart: [osd.0] - - sleep: - duration: 30 - - ceph.restart: [osd.1] - - sleep: - duration: 30 - - ceph.restart: [osd.2] - - sleep: - duration: 30 - - ceph.restart: [osd.3] - - sleep: - duration: 30 - - ceph.restart: [osd.4] - - sleep: - duration: 30 - - ceph.restart: - daemons: [osd.5] - wait-for-healthy: false - wait-for-up-osds: true - - exec: - mgr.x: - - mkdir -p /var/lib/ceph/mgr/ceph-x - - ceph auth get-or-create-key mgr.x mon 'allow profile mgr' - - ceph auth export mgr.x > /var/lib/ceph/mgr/ceph-x/keyring - - ceph.restart: - daemons: [mgr.x] - wait-for-healthy: false - - exec: - osd.0: - - ceph osd require-osd-release luminous - - ceph.healthy: - - print: "**** done ceph.restart all -x branch mds/osd/mon" diff --git a/ceph/qa/suites/upgrade/luminous-x/point-to-point-x/point-to-point-upgrade.yaml b/ceph/qa/suites/upgrade/luminous-x/point-to-point-x/point-to-point-upgrade.yaml index 717c4d0ba..0a4ed9ac8 100644 --- a/ceph/qa/suites/upgrade/luminous-x/point-to-point-x/point-to-point-upgrade.yaml +++ b/ceph/qa/suites/upgrade/luminous-x/point-to-point-x/point-to-point-upgrade.yaml @@ -109,7 +109,7 @@ workload: workload_luminous: full_sequential: - workunit: - branch: luminous + tag: v12.2.2 clients: client.1: - rados/test.sh @@ -166,14 +166,14 @@ workload_x: branch: luminous clients: client.1: - - rados/test.sh + - rados/test-upgrade-to-mimic.sh - cls - - print: "**** done rados/test.sh & cls workload_x NOT upgraded client" + - print: "**** done rados/test-upgrade-to-mimic.sh & cls workload_x NOT upgraded client" - workunit: branch: luminous clients: client.0: - - rados/test.sh + - rados/test-upgrade-to-mimic.sh - cls - print: "**** done rados/test.sh & cls workload_x upgraded client" - rgw: [client.1] diff --git a/ceph/qa/tasks/cephfs/filesystem.py b/ceph/qa/tasks/cephfs/filesystem.py index 402d6d120..6cc1ea558 100644 --- a/ceph/qa/tasks/cephfs/filesystem.py +++ b/ceph/qa/tasks/cephfs/filesystem.py @@ -1051,7 +1051,8 @@ class Filesystem(MDSCluster): else: return True - def rados(self, args, pool=None, namespace=None, stdin_data=None): + def rados(self, args, pool=None, namespace=None, stdin_data=None, + stdin_file=None): """ Call into the `rados` CLI from an MDS """ @@ -1069,6 +1070,10 @@ class Filesystem(MDSCluster): args = ([os.path.join(self._prefix, "rados"), "-p", pool] + (["--namespace", namespace] if namespace else []) + args) + + if stdin_file is not None: + args = ["bash", "-c", "cat " + stdin_file + " | " + " ".join(args)] + p = remote.run( args=args, stdin=stdin_data, diff --git a/ceph/qa/tasks/cephfs/test_full.py b/ceph/qa/tasks/cephfs/test_full.py index 867ac0ef3..9395612e4 100644 --- a/ceph/qa/tasks/cephfs/test_full.py +++ b/ceph/qa/tasks/cephfs/test_full.py @@ -250,7 +250,7 @@ class FullnessTestCase(CephFSTestCase): print "writing some data through which we expect to succeed" bytes = 0 f = os.open("{file_path}", os.O_WRONLY | os.O_CREAT) - bytes += os.write(f, 'a' * 4096) + bytes += os.write(f, 'a' * 512 * 1024) os.fsync(f) print "fsync'ed data successfully, will now attempt to fill fs" @@ -260,10 +260,10 @@ class FullnessTestCase(CephFSTestCase): # from write full = False - for n in range(0, {fill_mb}): + for n in range(0, int({fill_mb} * 0.9)): bytes += os.write(f, 'x' * 1024 * 1024) - print "wrote bytes via buffered write, may repeat" - print "done writing bytes" + print "wrote {{0}} bytes via buffered write, may repeat".format(bytes) + print "done writing {{0}} bytes".format(bytes) # OK, now we should sneak in under the full condition # due to the time it takes the OSDs to report to the @@ -271,13 +271,16 @@ class FullnessTestCase(CephFSTestCase): os.fsync(f) print "successfully fsync'ed prior to getting full state reported" - # Now wait for the full flag to get set so that our - # next flush IO will fail - time.sleep(30) + # buffered write, add more dirty data to the buffer + print "starting buffered write" + try: + for n in range(0, int({fill_mb} * 0.2)): + bytes += os.write(f, 'x' * 1024 * 1024) + print "sleeping a bit as we've exceeded 90% of our expected full ratio" + time.sleep({full_wait}) + except OSError: + pass; - # A buffered IO, should succeed - print "starting buffered write we expect to succeed" - os.write(f, 'x' * 4096) print "wrote, now waiting 30s and then doing a close we expect to fail" # Wait long enough for a background flush that should fail @@ -328,7 +331,7 @@ class FullnessTestCase(CephFSTestCase): # from write full = False - for n in range(0, {fill_mb} + 1): + for n in range(0, int({fill_mb} * 1.1)): try: bytes += os.write(f, 'x' * 1024 * 1024) print "wrote bytes via buffered write, moving on to fsync" @@ -346,10 +349,10 @@ class FullnessTestCase(CephFSTestCase): else: print "Not full yet after %.2f MB" % (bytes / (1024.0 * 1024.0)) - if n > {fill_mb} * 0.8: + if n > {fill_mb} * 0.9: # Be cautious in the last region where we expect to hit # the full condition, so that we don't overshoot too dramatically - print "sleeping a bit as we've exceeded 80% of our expected full ratio" + print "sleeping a bit as we've exceeded 90% of our expected full ratio" time.sleep({full_wait}) if not full: @@ -401,11 +404,10 @@ class TestClusterFull(FullnessTestCase): super(TestClusterFull, self).setUp() if self.pool_capacity is None: - # This is a hack to overcome weird fluctuations in the reported - # `max_avail` attribute of pools that sometimes occurs in between - # tests (reason as yet unclear, but this dodges the issue) - TestClusterFull.pool_capacity = self.fs.get_pool_df(self._data_pool_name())['max_avail'] - TestClusterFull.fill_mb = int(1.05 * (self.pool_capacity / (1024.0 * 1024.0))) + max_avail = self.fs.get_pool_df(self._data_pool_name())['max_avail'] + full_ratio = float(self.fs.get_config("mon_osd_full_ratio", service_type="mon")) + TestClusterFull.pool_capacity = int(max_avail * full_ratio) + TestClusterFull.fill_mb = (self.pool_capacity / (1024 * 1024)) def is_full(self): return self.fs.is_full() diff --git a/ceph/qa/tasks/cephfs/test_scrub.py b/ceph/qa/tasks/cephfs/test_scrub.py new file mode 100644 index 000000000..32371dd67 --- /dev/null +++ b/ceph/qa/tasks/cephfs/test_scrub.py @@ -0,0 +1,157 @@ +""" +Test CephFS scrub (distinct from OSD scrub) functionality +""" +import logging +import os +import traceback +from collections import namedtuple + +from teuthology.orchestra.run import CommandFailedError +from tasks.cephfs.cephfs_test_case import CephFSTestCase, for_teuthology + +log = logging.getLogger(__name__) + +ValidationError = namedtuple("ValidationError", ["exception", "backtrace"]) + + +class Workload(object): + def __init__(self, filesystem, mount): + self._mount = mount + self._filesystem = filesystem + self._initial_state = None + + # Accumulate backtraces for every failed validation, and return them. Backtraces + # are rather verbose, but we only see them when something breaks, and they + # let us see which check failed without having to decorate each check with + # a string + self._errors = [] + + def assert_equal(self, a, b): + try: + if a != b: + raise AssertionError("{0} != {1}".format(a, b)) + except AssertionError as e: + self._errors.append( + ValidationError(e, traceback.format_exc(3)) + ) + + def write(self): + """ + Write the workload files to the mount + """ + raise NotImplementedError() + + def validate(self): + """ + Read from the mount and validate that the workload files are present (i.e. have + survived or been reconstructed from the test scenario) + """ + raise NotImplementedError() + + def damage(self): + """ + Damage the filesystem pools in ways that will be interesting to recover from. By + default just wipe everything in the metadata pool + """ + # Delete every object in the metadata pool + objects = self._filesystem.rados(["ls"]).split("\n") + for o in objects: + self._filesystem.rados(["rm", o]) + + def flush(self): + """ + Called after client unmount, after write: flush whatever you want + """ + self._filesystem.mds_asok(["flush", "journal"]) + + +class BacktraceWorkload(Workload): + """ + Single file, single directory, wipe the backtrace and check it. + """ + def write(self): + self._mount.run_shell(["mkdir", "subdir"]) + self._mount.write_n_mb("subdir/sixmegs", 6) + + def validate(self): + st = self._mount.stat("subdir/sixmegs") + self._filesystem.mds_asok(["flush", "journal"]) + bt = self._filesystem.read_backtrace(st['st_ino']) + parent = bt['ancestors'][0]['dname'] + self.assert_equal(parent, "sixmegs") + return self._errors + + def damage(self): + st = self._mount.stat("subdir/sixmegs") + self._filesystem.mds_asok(["flush", "journal"]) + self._filesystem._write_data_xattr(st['st_ino'], "parent", "") + + +class DupInodeWorkload(Workload): + """ + Duplicate an inode and try scrubbing it twice." + """ + + def write(self): + self._mount.run_shell(["mkdir", "parent"]) + self._mount.run_shell(["mkdir", "parent/child"]) + self._mount.write_n_mb("parent/parentfile", 6) + self._mount.write_n_mb("parent/child/childfile", 6) + + def damage(self): + temp_bin_path = "/tmp/10000000000.00000000_omap.bin" + self._mount.umount() + self._filesystem.mds_asok(["flush", "journal"]) + self._filesystem.mds_stop() + self._filesystem.rados(["getomapval", "10000000000.00000000", + "parentfile_head", temp_bin_path]) + self._filesystem.rados(["setomapval", "10000000000.00000000", + "shadow_head"], stdin_file=temp_bin_path) + self._filesystem.set_ceph_conf('mds', 'mds hack allow loading invalid metadata', True) + self._filesystem.mds_restart() + self._filesystem.wait_for_daemons() + + def validate(self): + self._filesystem.mds_asok(["scrub_path", "/", "recursive", "repair"]) + self.assert_equal(self._filesystem.are_daemons_healthy(), True) + return self._errors + + +class TestScrub(CephFSTestCase): + MDSS_REQUIRED = 1 + + def _scrub(self, workload, workers=1): + """ + That when all objects in metadata pool are removed, we can rebuild a metadata pool + based on the contents of a data pool, and a client can see and read our files. + """ + + # First, inject some files + + workload.write() + + # are off by default, but in QA we need to explicitly disable them) + self.fs.set_ceph_conf('mds', 'mds verify scatter', False) + self.fs.set_ceph_conf('mds', 'mds debug scatterstat', False) + + # Apply any data damage the workload wants + workload.damage() + + self.fs.mds_asok(["scrub_path", "/", "recursive", "repair"]) + + # See that the files are present and correct + errors = workload.validate() + if errors: + log.error("Validation errors found: {0}".format(len(errors))) + for e in errors: + log.error(e.exception) + log.error(e.backtrace) + raise AssertionError("Validation failed, first error: {0}\n{1}".format( + errors[0].exception, errors[0].backtrace + )) + + def test_scrub_backtrace(self): + self._scrub(BacktraceWorkload(self.fs, self.mount_a)) + + def test_scrub_dup_inode(self): + self._scrub(DupInodeWorkload(self.fs, self.mount_a)) diff --git a/ceph/qa/tasks/mgr/test_module_selftest.py b/ceph/qa/tasks/mgr/test_module_selftest.py index 5bb4602ae..b7fb57061 100644 --- a/ceph/qa/tasks/mgr/test_module_selftest.py +++ b/ceph/qa/tasks/mgr/test_module_selftest.py @@ -31,6 +31,7 @@ class TestModuleSelftest(MgrTestCase): self._selftest_plugin("zabbix") def test_prometheus(self): + self._assign_ports("prometheus", "server_port", min_port=8100) self._selftest_plugin("prometheus") def test_influx(self): diff --git a/ceph/qa/tasks/qemu.py b/ceph/qa/tasks/qemu.py index 82252e1e9..7a1abe8f5 100644 --- a/ceph/qa/tasks/qemu.py +++ b/ceph/qa/tasks/qemu.py @@ -7,6 +7,7 @@ import contextlib import logging import os import yaml +import time from teuthology import misc as teuthology from teuthology import contextutil @@ -407,6 +408,7 @@ def run_qemu(ctx, config): cachemode=cachemode, ), ]) + time_wait = client_config.get('time_wait', 0) log.info('starting qemu...') procs.append( @@ -424,6 +426,11 @@ def run_qemu(ctx, config): log.info('waiting for qemu tests to finish...') run.wait(procs) + if time_wait > 0: + log.debug('waiting {time_wait} sec for workloads detect finish...'.format( + time_wait=time_wait)); + time.sleep(time_wait) + log.debug('checking that qemu tests succeeded...') for client in config.iterkeys(): (remote,) = ctx.cluster.only(client).remotes.keys() diff --git a/ceph/qa/tasks/swift.py b/ceph/qa/tasks/swift.py index 28f75dd0a..9cd580a02 100644 --- a/ceph/qa/tasks/swift.py +++ b/ceph/qa/tasks/swift.py @@ -28,7 +28,10 @@ def download(ctx, config): for client in config: ctx.cluster.only(client).run( args=[ - 'git', 'clone', + 'git', + 'clone', + '--branch', + 'ceph-luminous', teuth_config.ceph_git_base_url + 'swift.git', '{tdir}/swift'.format(tdir=testdir), ], diff --git a/ceph/qa/workunits/rados/test-upgrade-to-mimic.sh b/ceph/qa/workunits/rados/test-upgrade-to-mimic.sh new file mode 100755 index 000000000..ad18c50a4 --- /dev/null +++ b/ceph/qa/workunits/rados/test-upgrade-to-mimic.sh @@ -0,0 +1,52 @@ +#!/bin/bash -ex + +parallel=1 +[ "$1" = "--serial" ] && parallel=0 + +color="" +[ -t 1 ] && color="--gtest_color=yes" + +function cleanup() { + pkill -P $$ || true +} +trap cleanup EXIT ERR HUP INT QUIT + +declare -A pids + +for f in \ + api_aio api_io api_list api_lock api_misc \ + api_tier api_pool api_snapshots api_stat api_watch_notify api_cmd \ + api_service \ + api_c_write_operations \ + 'api_c_read_operations --gtest_filter=-CReadOpsTest.Exec' \ + list_parallel \ + open_pools_parallel \ + delete_pools_parallel \ + watch_notify +do + if [ $parallel -eq 1 ]; then + r=`printf '%25s' $f` + ff=`echo $f | awk '{print $1}'` + bash -o pipefail -exc "ceph_test_rados_$f $color 2>&1 | tee ceph_test_rados_$ff.log | sed \"s/^/$r: /\"" & + pid=$! + echo "test $f on pid $pid" + pids[$f]=$pid + else + ceph_test_rados_$f + fi +done + +ret=0 +if [ $parallel -eq 1 ]; then +for t in "${!pids[@]}" +do + pid=${pids[$t]} + if ! wait $pid + then + echo "error in $t ($pid)" + ret=1 + fi +done +fi + +exit $ret diff --git a/ceph/qa/workunits/rbd/run_devstack_tempest.sh b/ceph/qa/workunits/rbd/run_devstack_tempest.sh index 065dbef78..7ee21f09f 100755 --- a/ceph/qa/workunits/rbd/run_devstack_tempest.sh +++ b/ceph/qa/workunits/rbd/run_devstack_tempest.sh @@ -1,6 +1,7 @@ #!/bin/bash -ex STACK_BRANCH=stable/pike +TEMPEST_BRANCH=17.2.0 STACK_USER=${STACK_USER:-stack} STACK_GROUP=${STACK_GROUP:-stack} @@ -102,6 +103,7 @@ sed -i 's/appdirs===1.4.0/appdirs===1.4.3/' requirements/upper-constraints.txt cd devstack cp ${STACK_HOME_PATH}/local.conf . +export TEMPEST_BRANCH=${TEMPEST_BRANCH} export PYTHONUNBUFFERED=true export PROJECTS="openstack/devstack-plugin-ceph openstack/devstack-plugin-mariadb" diff --git a/ceph/qa/workunits/rbd/test_admin_socket.sh b/ceph/qa/workunits/rbd/test_admin_socket.sh index a7ecd839c..5bff60355 100755 --- a/ceph/qa/workunits/rbd/test_admin_socket.sh +++ b/ceph/qa/workunits/rbd/test_admin_socket.sh @@ -112,8 +112,6 @@ function rbd_watch_end() $(rbd_watch_out_file ${image}) $(rbd_watch_asok ${image}) } -wait_for_clean - pool="rbd" image=testimg$$ ceph_admin="ceph --admin-daemon $(rbd_watch_asok ${image})" diff --git a/ceph/run-make-check.sh b/ceph/run-make-check.sh index aa8f8ed28..078345422 100755 --- a/ceph/run-make-check.sh +++ b/ceph/run-make-check.sh @@ -30,8 +30,8 @@ function get_processors() { function run() { local install_cmd local which_pkg="which" + source /etc/os-release if test -f /etc/redhat-release ; then - source /etc/os-release if ! type bc > /dev/null 2>&1 ; then echo "Please install bc and re-run." exit 1 @@ -41,13 +41,13 @@ function run() { else install_cmd="yum install -y" fi - else + elif type zypper > /dev/null 2>&1 ; then + install_cmd="zypper --gpg-auto-import-keys --non-interactive install --no-recommends" + elif type apt-get > /dev/null 2>&1 ; then + install_cmd="apt-get install -y" which_pkg="debianutils" fi - type apt-get > /dev/null 2>&1 && install_cmd="apt-get install -y" - type zypper > /dev/null 2>&1 && install_cmd="zypper --gpg-auto-import-keys --non-interactive install" - if ! type sudo > /dev/null 2>&1 ; then echo "Please install sudo and re-run. This script assumes it is running" echo "as a normal user with the ability to run commands as root via sudo." @@ -57,6 +57,7 @@ function run() { $DRY_RUN sudo $install_cmd ccache jq $which_pkg else echo "WARNING: Don't know how to install packages" >&2 + echo "This probably means distribution $ID is not supported by run-make-check.sh" >&2 fi if test -f ./install-deps.sh ; then diff --git a/ceph/src/.git_version b/ceph/src/.git_version index cbdd56899..823f28dde 100644 --- a/ceph/src/.git_version +++ b/ceph/src/.git_version @@ -1,2 +1,2 @@ -52085d5249a80c5f5121a76d6288429f35e4e77b -v12.2.4 +cad919881333ac92274171586c827e01f554a70a +v12.2.5 diff --git a/ceph/src/ceph-detect-init/CMakeLists.txt b/ceph/src/ceph-detect-init/CMakeLists.txt index 066be4150..3d4aaf36d 100644 --- a/ceph/src/ceph-detect-init/CMakeLists.txt +++ b/ceph/src/ceph-detect-init/CMakeLists.txt @@ -3,7 +3,7 @@ set(CEPH_DETECT_INIT_VIRTUALENV ${CEPH_BUILD_VIRTUALENV}/ceph-detect-init-virtua add_custom_target(ceph-detect-init COMMAND ${CMAKE_SOURCE_DIR}/src/tools/setup-virtualenv.sh ${CEPH_DETECT_INIT_VIRTUALENV} && - ${CEPH_DETECT_INIT_VIRTUALENV}/bin/pip install --no-index --use-wheel --find-links=file:${CMAKE_SOURCE_DIR}/src/ceph-detect-init/wheelhouse -e . + ${CEPH_DETECT_INIT_VIRTUALENV}/bin/pip install --no-index --find-links=file:${CMAKE_SOURCE_DIR}/src/ceph-detect-init/wheelhouse -e . WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src/ceph-detect-init COMMENT "ceph-detect-init is being created") add_dependencies(tests ceph-detect-init) diff --git a/ceph/src/ceph-detect-init/tox.ini b/ceph/src/ceph-detect-init/tox.ini index dd631a27d..54691f78d 100644 --- a/ceph/src/ceph-detect-init/tox.ini +++ b/ceph/src/ceph-detect-init/tox.ini @@ -10,7 +10,6 @@ setenv = VIRTUAL_ENV={envdir} usedevelop = true deps = {env:NO_INDEX:} - --use-wheel --find-links=file://{toxinidir}/wheelhouse -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt diff --git a/ceph/src/ceph-disk/CMakeLists.txt b/ceph/src/ceph-disk/CMakeLists.txt index 1bce4d416..c6bd09a12 100644 --- a/ceph/src/ceph-disk/CMakeLists.txt +++ b/ceph/src/ceph-disk/CMakeLists.txt @@ -3,7 +3,7 @@ set(CEPH_DISK_VIRTUALENV ${CEPH_BUILD_VIRTUALENV}/ceph-disk-virtualenv) add_custom_target(ceph-disk COMMAND ${CMAKE_SOURCE_DIR}/src/tools/setup-virtualenv.sh ${CEPH_DISK_VIRTUALENV} && - ${CEPH_DISK_VIRTUALENV}/bin/pip install --no-index --use-wheel --find-links=file:${CMAKE_SOURCE_DIR}/src/ceph-disk/wheelhouse -e . + ${CEPH_DISK_VIRTUALENV}/bin/pip install --no-index --find-links=file:${CMAKE_SOURCE_DIR}/src/ceph-disk/wheelhouse -e . WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src/ceph-disk COMMENT "ceph-disk is being created") add_dependencies(tests ceph-disk) diff --git a/ceph/src/ceph-disk/tox.ini b/ceph/src/ceph-disk/tox.ini index c65f52a83..be75a4500 100644 --- a/ceph/src/ceph-disk/tox.ini +++ b/ceph/src/ceph-disk/tox.ini @@ -10,14 +10,13 @@ setenv = usedevelop = true deps = {env:NO_INDEX:} - --use-wheel --find-links=file://{toxinidir}/wheelhouse -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt ../ceph-detect-init [testenv:py27] -sitepackages=True +basepython = python2.7 passenv = CEPH_ROOT CEPH_BIN CEPH_LIB CEPH_BUILD_VIRTUALENV changedir = {env:CEPH_BUILD_DIR} commands = coverage run --append --source=ceph_disk {envbindir}/py.test -vv {toxinidir}/tests/test_main.py diff --git a/ceph/src/ceph-volume/ceph_volume/api/lvm.py b/ceph/src/ceph-volume/ceph_volume/api/lvm.py index 656c19ddf..88f0d2d2a 100644 --- a/ceph/src/ceph-volume/ceph_volume/api/lvm.py +++ b/ceph/src/ceph-volume/ceph_volume/api/lvm.py @@ -3,9 +3,13 @@ API for CRUD lvm tag operations. Follows the Ceph LVM tag naming convention that prefixes tags with ``ceph.`` and uses ``=`` for assignment, and provides set of utilities for interacting with LVM. """ +import logging +import os from ceph_volume import process from ceph_volume.exceptions import MultipleLVsError, MultipleVGsError, MultiplePVsError +logger = logging.getLogger(__name__) + def _output_parser(output, fields): """ @@ -71,14 +75,106 @@ def parse_tags(lv_tags): return tag_mapping +def _vdo_parents(devices): + """ + It is possible we didn't get a logical volume, or a mapper path, but + a device like /dev/sda2, to resolve this, we must look at all the slaves of + every single device in /sys/block and if any of those devices is related to + VDO devices, then we can add the parent + """ + parent_devices = [] + for parent in os.listdir('/sys/block'): + for slave in os.listdir('/sys/block/%s/slaves' % parent): + if slave in devices: + parent_devices.append('/dev/%s' % parent) + parent_devices.append(parent) + return parent_devices + + +def _vdo_slaves(vdo_names): + """ + find all the slaves associated with each vdo name (from realpath) by going + into /sys/block//slaves + """ + devices = [] + for vdo_name in vdo_names: + mapper_path = '/dev/mapper/%s' % vdo_name + if not os.path.exists(mapper_path): + continue + # resolve the realpath and realname of the vdo mapper + vdo_realpath = os.path.realpath(mapper_path) + vdo_realname = vdo_realpath.split('/')[-1] + slaves_path = '/sys/block/%s/slaves' % vdo_realname + if not os.path.exists(slaves_path): + continue + devices.append(vdo_realpath) + devices.append(mapper_path) + devices.append(vdo_realname) + for slave in os.listdir(slaves_path): + devices.append('/dev/%s' % slave) + devices.append(slave) + return devices + + +def _is_vdo(path): + """ + A VDO device can be composed from many different devices, go through each + one of those devices and its slaves (if any) and correlate them back to + /dev/mapper and their realpaths, and then check if they appear as part of + /sys/kvdo//statistics + + From the realpath of a logical volume, determine if it is a VDO device or + not, by correlating it to the presence of the name in + /sys/kvdo//statistics and all the previously captured devices + """ + if not os.path.isdir('/sys/kvdo'): + return False + realpath = os.path.realpath(path) + realpath_name = realpath.split('/')[-1] + devices = [] + vdo_names = set() + # get all the vdo names + for dirname in os.listdir('/sys/kvdo/'): + if os.path.isdir('/sys/kvdo/%s/statistics' % dirname): + vdo_names.add(dirname) + + # find all the slaves associated with each vdo name (from realpath) by + # going into /sys/block//slaves + devices.extend(_vdo_slaves(vdo_names)) + + # Find all possible parents, looking into slaves that are related to VDO + devices.extend(_vdo_parents(devices)) + + return any([ + path in devices, + realpath in devices, + realpath_name in devices]) + + +def is_vdo(path): + """ + Detect if a path is backed by VDO, proxying the actual call to _is_vdo so + that we can prevent an exception breaking OSD creation. If an exception is + raised, it will get captured and logged to file, while returning + a ``False``. + """ + try: + if _is_vdo(path): + return '1' + return '0' + except Exception: + logger.exception('Unable to properly detect device as VDO: %s', path) + return '0' + + def get_api_vgs(): """ Return the list of group volumes available in the system using flags to include common metadata associated with them - Command and sample delimeted output, should look like:: + Command and sample delimited output should look like:: - $ vgs --noheadings --separator=';' \ + $ vgs --noheadings --readonly --separator=';' \ -o vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free ubuntubox-vg;1;2;0;wz--n-;299.52g;12.00m osd_vg;3;1;0;wz--n-;29.21g;9.21g @@ -86,7 +182,7 @@ def get_api_vgs(): """ fields = 'vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free' stdout, stderr, returncode = process.call( - ['vgs', '--noheadings', '--separator=";"', '-o', fields] + ['vgs', '--noheadings', '--readonly', '--separator=";"', '-o', fields] ) return _output_parser(stdout, fields) @@ -96,16 +192,16 @@ def get_api_lvs(): Return the list of logical volumes available in the system using flags to include common metadata associated with them - Command and delimeted output, should look like:: + Command and delimited output should look like:: - $ lvs --noheadings --separator=';' -o lv_tags,lv_path,lv_name,vg_name + $ lvs --noheadings --readonly --separator=';' -o lv_tags,lv_path,lv_name,vg_name ;/dev/ubuntubox-vg/root;root;ubuntubox-vg ;/dev/ubuntubox-vg/swap_1;swap_1;ubuntubox-vg """ fields = 'lv_tags,lv_path,lv_name,vg_name,lv_uuid' stdout, stderr, returncode = process.call( - ['lvs', '--noheadings', '--separator=";"', '-o', fields] + ['lvs', '--noheadings', '--readonly', '--separator=";"', '-o', fields] ) return _output_parser(stdout, fields) @@ -117,9 +213,9 @@ def get_api_pvs(): This will only return physical volumes set up to work with LVM. - Command and delimeted output, should look like:: + Command and delimited output should look like:: - $ pvs --noheadings --separator=';' -o pv_name,pv_tags,pv_uuid + $ pvs --noheadings --readonly --separator=';' -o pv_name,pv_tags,pv_uuid /dev/sda1;; /dev/sdv;;07A4F654-4162-4600-8EB3-88D1E42F368D @@ -127,7 +223,7 @@ def get_api_pvs(): fields = 'pv_name,pv_tags,pv_uuid,vg_name' stdout, stderr, returncode = process.call( - ['pvs', '--no-heading', '--separator=";"', '-o', fields] + ['pvs', '--no-heading', '--readonly', '--separator=";"', '-o', fields] ) return _output_parser(stdout, fields) @@ -217,7 +313,7 @@ def remove_vg(vg_name): """ Removes a volume group. """ - fail_msg = "Unable to remove vg %s".format(vg_name) + fail_msg = "Unable to remove vg %s" % vg_name process.run( [ 'vgremove', @@ -233,7 +329,7 @@ def remove_pv(pv_name): """ Removes a physical volume. """ - fail_msg = "Unable to remove vg %s".format(pv_name) + fail_msg = "Unable to remove vg %s" % pv_name process.run( [ 'pvremove', @@ -263,7 +359,7 @@ def remove_lv(path): terminal_verbose=True, ) if returncode != 0: - raise RuntimeError("Unable to remove %s".format(path)) + raise RuntimeError("Unable to remove %s" % path) return True @@ -446,7 +542,7 @@ class Volumes(list): def _purge(self): """ - Deplete all the items in the list, used internally only so that we can + Delete all the items in the list, used internally only so that we can dynamically allocate the items when filtering without the concern of messing up the contents """ diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py index 81ebe6b0b..37ec6fb72 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py @@ -9,17 +9,19 @@ from ceph_volume.util import prepare as prepare_utils from ceph_volume.util import encryption as encryption_utils from ceph_volume.systemd import systemctl from ceph_volume.api import lvm as api +from .listing import direct_report logger = logging.getLogger(__name__) -def activate_filestore(lvs): +def activate_filestore(lvs, no_systemd=False): # find the osd osd_lv = lvs.get(lv_tags={'ceph.type': 'data'}) if not osd_lv: raise RuntimeError('Unable to find a data LV for filestore activation') is_encrypted = osd_lv.tags.get('ceph.encrypted', '0') == '1' + is_vdo = osd_lv.tags.get('ceph.vdo', '0') osd_id = osd_lv.tags['ceph.osd_id'] conf.cluster = osd_lv.tags['ceph.cluster_name'] @@ -54,10 +56,11 @@ def activate_filestore(lvs): source = '/dev/mapper/%s' % osd_lv.lv_uuid else: source = osd_lv.lv_path + # mount the osd destination = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id) if not system.device_is_mounted(source, destination=destination): - process.run(['mount', '-v', source, destination]) + prepare_utils.mount_osd(source, osd_id, is_vdo=is_vdo) # always re-do the symlink regardless if it exists, so that the journal # device path that may have changed can be mapped correctly every time @@ -67,11 +70,12 @@ def activate_filestore(lvs): # make sure that the journal has proper permissions system.chown(osd_journal) - # enable the ceph-volume unit for this OSD - systemctl.enable_volume(osd_id, osd_fsid, 'lvm') + if no_systemd is False: + # enable the ceph-volume unit for this OSD + systemctl.enable_volume(osd_id, osd_fsid, 'lvm') - # start the OSD - systemctl.start_osd(osd_id) + # start the OSD + systemctl.start_osd(osd_id) terminal.success("ceph-volume lvm activate successful for osd ID: %s" % osd_id) @@ -108,9 +112,11 @@ def get_osd_device_path(osd_lv, lvs, device_type, dmcrypt_secret=None): return None -def activate_bluestore(lvs): +def activate_bluestore(lvs, no_systemd=False): # find the osd osd_lv = lvs.get(lv_tags={'ceph.type': 'block'}) + if not osd_lv: + raise RuntimeError('could not find a bluestore OSD to activate') is_encrypted = osd_lv.tags.get('ceph.encrypted', '0') == '1' dmcrypt_secret = None osd_id = osd_lv.tags['ceph.osd_id'] @@ -161,11 +167,12 @@ def activate_bluestore(lvs): process.run(['ln', '-snf', wal_device_path, destination]) system.chown(wal_device_path) - # enable the ceph-volume unit for this OSD - systemctl.enable_volume(osd_id, osd_fsid, 'lvm') + if no_systemd is False: + # enable the ceph-volume unit for this OSD + systemctl.enable_volume(osd_id, osd_fsid, 'lvm') - # start the OSD - systemctl.start_osd(osd_id) + # start the OSD + systemctl.start_osd(osd_id) terminal.success("ceph-volume lvm activate successful for osd ID: %s" % osd_id) @@ -177,15 +184,48 @@ class Activate(object): self.argv = argv @decorators.needs_root - def activate(self, args): + def activate_all(self, args): + listed_osds = direct_report() + osds = {} + for osd_id, devices in listed_osds.items(): + # the metadata for all devices in each OSD will contain + # the FSID which is required for activation + for device in devices: + fsid = device.get('tags', {}).get('ceph.osd_fsid') + if fsid: + osds[fsid] = osd_id + break + if not osds: + terminal.warning('Was unable to find any OSDs to activate') + terminal.warning('Verify OSDs are present with "ceph-volume lvm list"') + return + for osd_fsid, osd_id in osds.items(): + if systemctl.osd_is_active(osd_id): + terminal.warning( + 'OSD ID %s FSID %s process is active. Skipping activation' % (osd_id, osd_fsid) + ) + else: + terminal.info('Activating OSD ID %s FSID %s' % (osd_id, osd_fsid)) + self.activate(args, osd_id=osd_id, osd_fsid=osd_fsid) + + @decorators.needs_root + def activate(self, args, osd_id=None, osd_fsid=None): + """ + :param args: The parsed arguments coming from the CLI + :param osd_id: When activating all, this gets populated with an existing OSD ID + :param osd_fsid: When activating all, this gets populated with an existing OSD FSID + """ + osd_id = osd_id if osd_id is not None else args.osd_id + osd_fsid = osd_fsid if osd_fsid is not None else args.osd_fsid + lvs = api.Volumes() # filter them down for the OSD ID and FSID we need to activate - if args.osd_id and args.osd_fsid: - lvs.filter(lv_tags={'ceph.osd_id': args.osd_id, 'ceph.osd_fsid': args.osd_fsid}) - elif args.osd_fsid and not args.osd_id: - lvs.filter(lv_tags={'ceph.osd_fsid': args.osd_fsid}) + if osd_id and osd_fsid: + lvs.filter(lv_tags={'ceph.osd_id': osd_id, 'ceph.osd_fsid': osd_fsid}) + elif osd_fsid and not osd_id: + lvs.filter(lv_tags={'ceph.osd_fsid': osd_fsid}) if not lvs: - raise RuntimeError('could not find osd.%s with fsid %s' % (args.osd_id, args.osd_fsid)) + raise RuntimeError('could not find osd.%s with fsid %s' % (osd_id, osd_fsid)) # This argument is only available when passed in directly or via # systemd, not when ``create`` is being used if getattr(args, 'auto_detect_objectstore', False): @@ -199,10 +239,9 @@ class Activate(object): logger.info('unable to find a journal associated with the OSD, assuming bluestore') return activate_bluestore(lvs) if args.bluestore: - activate_bluestore(lvs) + activate_bluestore(lvs, no_systemd=args.no_systemd) elif args.filestore: - activate_filestore(lvs) - terminal.success("ceph-volume lvm activate successful for osd ID: %s" % args.osd_id) + activate_filestore(lvs, no_systemd=args.no_systemd) def main(self): sub_command_help = dedent(""" @@ -214,6 +253,11 @@ class Activate(object): The lvs associated with the OSD need to have been prepared previously, so that all needed tags and metadata exist. + When migrating OSDs, or a multiple-osd activation is needed, the + ``--all`` flag can be used instead of the individual ID and FSID: + + ceph-volume lvm activate --all + """) parser = argparse.ArgumentParser( prog='ceph-volume lvm activate', @@ -241,12 +285,24 @@ class Activate(object): parser.add_argument( '--bluestore', action='store_true', - help='filestore objectstore (not yet implemented)', + help='bluestore objectstore (default)', ) parser.add_argument( '--filestore', action='store_true', - help='filestore objectstore (current default)', + help='filestore objectstore', + ) + parser.add_argument( + '--all', + dest='activate_all', + action='store_true', + help='Activate all OSDs found in the system', + ) + parser.add_argument( + '--no-systemd', + dest='no_systemd', + action='store_true', + help='Skip creating and enabling systemd units and starting OSD services', ) if len(self.argv) == 0: print(sub_command_help) @@ -256,4 +312,7 @@ class Activate(object): # cause both to be True if not args.bluestore and not args.filestore: args.bluestore = True - self.activate(args) + if args.activate_all: + self.activate_all(args) + else: + self.activate(args) diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/common.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/common.py index 869fcf200..332398972 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/common.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/common.py @@ -111,6 +111,13 @@ def common_parser(prog, description): help='Enable device encryption via dm-crypt', ) + parser.add_argument( + '--no-systemd', + dest='no_systemd', + action='store_true', + help='Skip creating and enabling systemd units and starting OSD services when activating', + ) + # Do not parse args, so that consumers can do something before the args get # parsed triggering argparse behavior return parser diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/listing.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/listing.py index 6982f91bc..d1d96d7ad 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/listing.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/listing.py @@ -6,6 +6,7 @@ from textwrap import dedent from ceph_volume import decorators from ceph_volume.util import disk from ceph_volume.api import lvm as api +from ceph_volume.exceptions import MultipleLVsError logger = logging.getLogger(__name__) @@ -51,6 +52,21 @@ def pretty_report(report): print(''.join(output)) +def direct_report(): + """ + Other non-cli consumers of listing information will want to consume the + report without the need to parse arguments or other flags. This helper + bypasses the need to deal with the class interface which is meant for cli + handling. + """ + _list = List([]) + # this is crucial: make sure that all paths will reflect current + # information. In the case of a system that has migrated, the disks will + # have changed paths + _list.update() + return _list.full_report() + + class List(object): help = 'list logical volumes and devices associated with Ceph' @@ -119,11 +135,22 @@ class List(object): """ Generate a report for a single device. This can be either a logical volume in the form of vg/lv or a device with an absolute path like - /dev/sda1 + /dev/sda1 or /dev/sda """ lvs = api.Volumes() report = {} lv = api.get_lv_from_argument(device) + + # check if there was a pv created with the + # name of device + pv = api.get_pv(pv_name=device) + if pv and not lv: + try: + lv = api.get_lv(vg_name=pv.vg_name) + except MultipleLVsError: + lvs.filter(vg_name=pv.vg_name) + return self.full_report(lvs=lvs) + if lv: try: _id = lv.tags['ceph.osd_id'] @@ -158,12 +185,13 @@ class List(object): ) return report - def full_report(self): + def full_report(self, lvs=None): """ Generate a report for all the logical volumes and associated devices that have been previously prepared by Ceph """ - lvs = api.Volumes() + if lvs is None: + lvs = api.Volumes() report = {} for lv in lvs: try: diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/prepare.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/prepare.py index 360cf04ac..17ac5e1b3 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/prepare.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/prepare.py @@ -56,12 +56,14 @@ def prepare_filestore(device, journal, secrets, tags, osd_id, fsid): device = prepare_dmcrypt(key, device, 'data', tags) journal = prepare_dmcrypt(key, journal, 'journal', tags) + # vdo detection + is_vdo = api.is_vdo(device) # create the directory prepare_utils.create_osd_path(osd_id) # format the device prepare_utils.format_device(device) # mount the data device - prepare_utils.mount_osd(device, osd_id) + prepare_utils.mount_osd(device, osd_id, is_vdo=is_vdo) # symlink the journal prepare_utils.link_journal(journal, osd_id) # get the latest monmap @@ -160,6 +162,7 @@ class Prepare(object): if device_name is None: return '', '', tags tags['ceph.type'] = device_type + tags['ceph.vdo'] = api.is_vdo(device_name) lv = self.get_lv(device_name) if lv: uuid = lv.lv_uuid @@ -189,11 +192,7 @@ class Prepare(object): """ if disk.is_partition(arg) or disk.is_device(arg): # we must create a vg, and then a single lv - vg_name = "ceph-%s" % cluster_fsid - if api.get_vg(vg_name=vg_name): - # means we already have a group for this, make a different one - # XXX this could end up being annoying for an operator, maybe? - vg_name = "ceph-%s" % str(uuid.uuid4()) + vg_name = "ceph-%s" % str(uuid.uuid4()) api.create_vg(vg_name, arg) lv_name = "osd-%s-%s" % (device_type, osd_fsid) return api.create_lv( @@ -263,6 +262,7 @@ class Prepare(object): tags['ceph.data_uuid'] = data_lv.lv_uuid tags['ceph.cephx_lockbox_secret'] = cephx_lockbox_secret tags['ceph.encrypted'] = encrypted + tags['ceph.vdo'] = api.is_vdo(data_lv.lv_path) journal_device, journal_uuid, tags = self.setup_device('journal', args.journal, tags) @@ -286,6 +286,7 @@ class Prepare(object): tags['ceph.block_uuid'] = block_lv.lv_uuid tags['ceph.cephx_lockbox_secret'] = cephx_lockbox_secret tags['ceph.encrypted'] = encrypted + tags['ceph.vdo'] = api.is_vdo(block_lv.lv_path) wal_device, wal_uuid, tags = self.setup_device('wal', args.block_wal, tags) db_device, db_uuid, tags = self.setup_device('db', args.block_db, tags) diff --git a/ceph/src/ceph-volume/ceph_volume/process.py b/ceph/src/ceph-volume/ceph_volume/process.py index 1a2c268db..9eece1241 100644 --- a/ceph/src/ceph-volume/ceph_volume/process.py +++ b/ceph/src/ceph-volume/ceph_volume/process.py @@ -9,6 +9,15 @@ import logging logger = logging.getLogger(__name__) +def which(executable): + """ + Proxy function to ceph_volume.util.system.which because the ``system`` + module does import ``process`` + """ + from ceph_volume.util import system + return system.which(executable) + + def log_output(descriptor, message, terminal_logging, logfile_logging): """ log output to both the logger and the terminal if terminal_logging is @@ -52,7 +61,7 @@ def log_descriptors(reads, process, terminal_logging): def obfuscate(command_, on=None): """ Certain commands that are useful to log might contain information that - should be replaced by '*' like when creating OSDs and the keyryings are + should be replaced by '*' like when creating OSDs and the keyrings are being passed, which should not be logged. :param on: A string (will match a flag) or an integer (will match an index) @@ -102,6 +111,8 @@ def run(command, **kw): stop_on_error = kw.pop('stop_on_error', True) command_msg = obfuscate(command, kw.pop('obfuscate', None)) fail_msg = kw.pop('fail_msg', None) + executable = which(command.pop(0)) + command.insert(0, executable) logger.info(command_msg) terminal.write(command_msg) terminal_logging = kw.pop('terminal_logging', True) @@ -162,10 +173,15 @@ def call(command, **kw): :param terminal_verbose: Log command output to terminal, defaults to False, and it is forcefully set to True if a return code is non-zero :param logfile_verbose: Log stderr/stdout output to log file. Defaults to True + :param verbose_on_failure: On a non-zero exit status, it will forcefully set logging ON for + the terminal. Defaults to True """ terminal_verbose = kw.pop('terminal_verbose', False) logfile_verbose = kw.pop('logfile_verbose', True) + verbose_on_failure = kw.pop('verbose_on_failure', True) show_command = kw.pop('show_command', False) + executable = which(command.pop(0)) + command.insert(0, executable) command_msg = "Running command: %s" % ' '.join(command) stdin = kw.pop('stdin', None) logger.info(command_msg) @@ -195,8 +211,11 @@ def call(command, **kw): if returncode != 0: # set to true so that we can log the stderr/stdout that callers would - # do anyway - terminal_verbose = True + # do anyway as long as verbose_on_failure is set (defaults to True) + if verbose_on_failure: + terminal_verbose = True + # logfiles aren't disruptive visually, unlike the terminal, so this + # should always be on when there is a failure logfile_verbose = True # the following can get a messed up order in the log if the system call diff --git a/ceph/src/ceph-volume/ceph_volume/systemd/systemctl.py b/ceph/src/ceph-volume/ceph_volume/systemd/systemctl.py index d5fd0dd80..688433774 100644 --- a/ceph/src/ceph-volume/ceph_volume/systemd/systemctl.py +++ b/ceph/src/ceph-volume/ceph_volume/systemd/systemctl.py @@ -24,6 +24,14 @@ def mask(unit): process.run(['systemctl', 'mask', unit]) +def is_active(unit): + out, err, rc = process.call( + ['systemctl', 'is-active', unit], + verbose_on_failure=False + ) + return rc == 0 + + def start_osd(id_): return start(osd_unit % id_) @@ -40,6 +48,10 @@ def disable_osd(id_): return disable(osd_unit % id_) +def osd_is_active(id_): + return is_active(osd_unit % id_) + + def enable_volume(id_, fsid, device_type='lvm'): return enable(volume_unit % (device_type, id_, fsid)) @@ -48,7 +60,7 @@ def mask_ceph_disk(): # systemctl allows using a glob like '*' for masking, but there was a bug # in that it wouldn't allow this for service templates. This means that # masking ceph-disk@* will not work, so we must link the service directly. - # /etc/systemd takes precendence regardless of the location of the unit + # /etc/systemd takes precedence regardless of the location of the unit process.run( ['ln', '-sf', '/dev/null', '/etc/systemd/system/ceph-disk@.service'] ) diff --git a/ceph/src/ceph-volume/ceph_volume/tests/api/test_lvm.py b/ceph/src/ceph-volume/ceph_volume/tests/api/test_lvm.py index a56582b35..adfa30c3b 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/api/test_lvm.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/api/test_lvm.py @@ -1,3 +1,4 @@ +import os import pytest from ceph_volume import process, exceptions from ceph_volume.api import lvm as api @@ -414,3 +415,149 @@ class TestCreateLV(object): api.create_lv('foo', 'foo_group', size='5G', tags={'ceph.type': 'data'}) data_tag = ['lvchange', '--addtag', 'ceph.data_device=/path', '/path'] assert capture.calls[2]['args'][0] == data_tag + + +# +# The following tests are pretty gnarly. VDO detection is very convoluted and +# involves correlating information from device mappers, realpaths, slaves of +# those mappers, and parents or related mappers. This makes it very hard to +# patch nicely or keep tests short and readable. These tests are trying to +# ensure correctness, the better approach will be to do some functional testing +# with VDO. +# + + +@pytest.fixture +def disable_kvdo_path(monkeypatch): + monkeypatch.setattr('os.path.isdir', lambda x: False) + + +@pytest.fixture +def enable_kvdo_path(monkeypatch): + monkeypatch.setattr('os.path.isdir', lambda x: True) + + +# Stub for os.listdir + + +class ListDir(object): + + def __init__(self, paths): + self.paths = paths + self._normalize_paths() + self.listdir = os.listdir + + def _normalize_paths(self): + for k, v in self.paths.items(): + self.paths[k.rstrip('/')] = v.rstrip('/') + + def add(self, original, fake): + self.paths[original.rstrip('/')] = fake.rstrip('/') + + def __call__(self, path): + return self.listdir(self.paths[path.rstrip('/')]) + + +@pytest.fixture(scope='function') +def listdir(monkeypatch): + def apply(paths=None, stub=None): + if not stub: + stub = ListDir(paths) + if paths: + for original, fake in paths.items(): + stub.add(original, fake) + + monkeypatch.setattr('os.listdir', stub) + return apply + + +@pytest.fixture(scope='function') +def makedirs(tmpdir): + def create(directory): + path = os.path.join(str(tmpdir), directory) + os.makedirs(path) + return path + create.base = str(tmpdir) + return create + + +class TestIsVdo(object): + + def test_no_vdo_dir(self, disable_kvdo_path): + assert api._is_vdo('/path') is False + + def test_exceptions_return_false(self, monkeypatch): + def throw(): + raise Exception() + monkeypatch.setattr('ceph_volume.api.lvm._is_vdo', throw) + assert api.is_vdo('/path') == '0' + + def test_is_vdo_returns_a_string(self, monkeypatch): + monkeypatch.setattr('ceph_volume.api.lvm._is_vdo', lambda x: True) + assert api.is_vdo('/path') == '1' + + def test_kvdo_dir_no_devices(self, makedirs, enable_kvdo_path, listdir, monkeypatch): + kvdo_path = makedirs('sys/kvdo') + listdir(paths={'/sys/kvdo': kvdo_path}) + monkeypatch.setattr('ceph_volume.api.lvm._vdo_slaves', lambda x: []) + monkeypatch.setattr('ceph_volume.api.lvm._vdo_parents', lambda x: []) + assert api._is_vdo('/dev/mapper/vdo0') is False + + def test_vdo_slaves_found_and_matched(self, makedirs, enable_kvdo_path, listdir, monkeypatch): + kvdo_path = makedirs('sys/kvdo') + listdir(paths={'/sys/kvdo': kvdo_path}) + monkeypatch.setattr('ceph_volume.api.lvm._vdo_slaves', lambda x: ['/dev/dm-3']) + monkeypatch.setattr('ceph_volume.api.lvm._vdo_parents', lambda x: []) + assert api._is_vdo('/dev/dm-3') is True + + def test_vdo_parents_found_and_matched(self, makedirs, enable_kvdo_path, listdir, monkeypatch): + kvdo_path = makedirs('sys/kvdo') + listdir(paths={'/sys/kvdo': kvdo_path}) + monkeypatch.setattr('ceph_volume.api.lvm._vdo_slaves', lambda x: []) + monkeypatch.setattr('ceph_volume.api.lvm._vdo_parents', lambda x: ['/dev/dm-4']) + assert api._is_vdo('/dev/dm-4') is True + + +class TestVdoSlaves(object): + + def test_slaves_are_not_found(self, makedirs, listdir, monkeypatch): + slaves_path = makedirs('sys/block/vdo0/slaves') + listdir(paths={'/sys/block/vdo0/slaves': slaves_path}) + monkeypatch.setattr('ceph_volume.api.lvm.os.path.exists', lambda x: True) + result = sorted(api._vdo_slaves(['vdo0'])) + assert '/dev/mapper/vdo0' in result + assert 'vdo0' in result + + def test_slaves_are_found(self, makedirs, listdir, monkeypatch): + slaves_path = makedirs('sys/block/vdo0/slaves') + makedirs('sys/block/vdo0/slaves/dm-4') + makedirs('dev/mapper/vdo0') + listdir(paths={'/sys/block/vdo0/slaves': slaves_path}) + monkeypatch.setattr('ceph_volume.api.lvm.os.path.exists', lambda x: True) + result = sorted(api._vdo_slaves(['vdo0'])) + assert '/dev/dm-4' in result + assert 'dm-4' in result + + +class TestVDOParents(object): + + def test_parents_are_found(self, makedirs, listdir): + block_path = makedirs('sys/block') + slaves_path = makedirs('sys/block/dm-4/slaves') + makedirs('sys/block/dm-4/slaves/dm-3') + listdir(paths={ + '/sys/block/dm-4/slaves': slaves_path, + '/sys/block': block_path}) + result = api._vdo_parents(['dm-3']) + assert '/dev/dm-4' in result + assert 'dm-4' in result + + def test_parents_are_not_found(self, makedirs, listdir): + block_path = makedirs('sys/block') + slaves_path = makedirs('sys/block/dm-4/slaves') + makedirs('sys/block/dm-4/slaves/dm-5') + listdir(paths={ + '/sys/block/dm-4/slaves': slaves_path, + '/sys/block': block_path}) + result = api._vdo_parents(['dm-3']) + assert result == [] diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py index ce623aac9..45d7b7191 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py @@ -1,6 +1,7 @@ import pytest from ceph_volume.devices.lvm import activate from ceph_volume.api import lvm as api +from ceph_volume.tests.conftest import Capture class Args(object): @@ -9,6 +10,7 @@ class Args(object): # default flags self.bluestore = False self.filestore = False + self.no_systemd = False self.auto_detect_objectstore = None for k, v in kw.items(): setattr(self, k, v) @@ -47,6 +49,102 @@ class TestActivate(object): with pytest.raises(RuntimeError): activate.Activate([]).activate(args) + def test_filestore_no_systemd(self, is_root, volumes, monkeypatch, capture): + fake_enable = Capture() + fake_start_osd = Capture() + monkeypatch.setattr('ceph_volume.util.system.device_is_mounted', lambda *a, **kw: True) + monkeypatch.setattr('ceph_volume.util.system.chown', lambda *a, **kw: True) + monkeypatch.setattr('ceph_volume.process.run', lambda *a, **kw: True) + monkeypatch.setattr(activate.systemctl, 'enable_volume', fake_enable) + monkeypatch.setattr(activate.systemctl, 'start_osd', fake_start_osd) + JournalVolume = api.Volume( + lv_name='journal', + lv_path='/dev/vg/journal', + lv_uuid='000', + lv_tags=','.join([ + "ceph.cluster_name=ceph", "ceph.journal_device=/dev/vg/journal", + "ceph.journal_uuid=000", "ceph.type=journal", + "ceph.osd_id=0","ceph.osd_fsid=1234"]) + ) + DataVolume = api.Volume( + lv_name='data', + lv_path='/dev/vg/data', + lv_tags="ceph.cluster_name=ceph,ceph.journal_device=/dev/vg/journal,ceph.journal_uuid=000,ceph.type=data,ceph.osd_id=0,ceph.osd_fsid=1234") + volumes.append(DataVolume) + volumes.append(JournalVolume) + monkeypatch.setattr(api, 'Volumes', lambda: volumes) + args = Args(osd_id=None, osd_fsid='1234', no_systemd=True, filestore=True) + activate.Activate([]).activate(args) + assert fake_enable.calls == [] + assert fake_start_osd.calls == [] + + def test_filestore_systemd(self, is_root, volumes, monkeypatch, capture): + fake_enable = Capture() + fake_start_osd = Capture() + monkeypatch.setattr('ceph_volume.util.system.device_is_mounted', lambda *a, **kw: True) + monkeypatch.setattr('ceph_volume.util.system.chown', lambda *a, **kw: True) + monkeypatch.setattr('ceph_volume.process.run', lambda *a, **kw: True) + monkeypatch.setattr(activate.systemctl, 'enable_volume', fake_enable) + monkeypatch.setattr(activate.systemctl, 'start_osd', fake_start_osd) + JournalVolume = api.Volume( + lv_name='journal', + lv_path='/dev/vg/journal', + lv_uuid='000', + lv_tags=','.join([ + "ceph.cluster_name=ceph", "ceph.journal_device=/dev/vg/journal", + "ceph.journal_uuid=000", "ceph.type=journal", + "ceph.osd_id=0","ceph.osd_fsid=1234"]) + ) + DataVolume = api.Volume( + lv_name='data', + lv_path='/dev/vg/data', + lv_tags="ceph.cluster_name=ceph,ceph.journal_device=/dev/vg/journal,ceph.journal_uuid=000,ceph.type=data,ceph.osd_id=0,ceph.osd_fsid=1234") + volumes.append(DataVolume) + volumes.append(JournalVolume) + monkeypatch.setattr(api, 'Volumes', lambda: volumes) + args = Args(osd_id=None, osd_fsid='1234', no_systemd=False, filestore=True) + activate.Activate([]).activate(args) + assert fake_enable.calls != [] + assert fake_start_osd.calls != [] + + def test_bluestore_no_systemd(self, is_root, volumes, monkeypatch, capture): + fake_enable = Capture() + fake_start_osd = Capture() + monkeypatch.setattr('ceph_volume.util.system.path_is_mounted', lambda *a, **kw: True) + monkeypatch.setattr('ceph_volume.util.system.chown', lambda *a, **kw: True) + monkeypatch.setattr('ceph_volume.process.run', lambda *a, **kw: True) + monkeypatch.setattr(activate.systemctl, 'enable_volume', fake_enable) + monkeypatch.setattr(activate.systemctl, 'start_osd', fake_start_osd) + DataVolume = api.Volume( + lv_name='data', + lv_path='/dev/vg/data', + lv_tags="ceph.cluster_name=ceph,,ceph.journal_uuid=000,ceph.type=block,ceph.osd_id=0,ceph.osd_fsid=1234") + volumes.append(DataVolume) + monkeypatch.setattr(api, 'Volumes', lambda: volumes) + args = Args(osd_id=None, osd_fsid='1234', no_systemd=True, bluestore=True) + activate.Activate([]).activate(args) + assert fake_enable.calls == [] + assert fake_start_osd.calls == [] + + def test_bluestore_systemd(self, is_root, volumes, monkeypatch, capture): + fake_enable = Capture() + fake_start_osd = Capture() + monkeypatch.setattr('ceph_volume.util.system.path_is_mounted', lambda *a, **kw: True) + monkeypatch.setattr('ceph_volume.util.system.chown', lambda *a, **kw: True) + monkeypatch.setattr('ceph_volume.process.run', lambda *a, **kw: True) + monkeypatch.setattr(activate.systemctl, 'enable_volume', fake_enable) + monkeypatch.setattr(activate.systemctl, 'start_osd', fake_start_osd) + DataVolume = api.Volume( + lv_name='data', + lv_path='/dev/vg/data', + lv_tags="ceph.cluster_name=ceph,,ceph.journal_uuid=000,ceph.type=block,ceph.osd_id=0,ceph.osd_fsid=1234") + volumes.append(DataVolume) + monkeypatch.setattr(api, 'Volumes', lambda: volumes) + args = Args(osd_id=None, osd_fsid='1234', no_systemd=False, bluestore=True) + activate.Activate([]).activate(args) + assert fake_enable.calls != [] + assert fake_start_osd.calls != [] + class TestActivateFlags(object): @@ -76,3 +174,93 @@ class TestActivateFlags(object): parsed_args = capture.calls[0]['args'][0] assert parsed_args.filestore is False assert parsed_args.bluestore is True + + +class TestActivateAll(object): + + def test_does_not_detect_osds(self, capsys, is_root, capture, monkeypatch): + monkeypatch.setattr('ceph_volume.devices.lvm.activate.direct_report', lambda: {}) + args = ['--all'] + activation = activate.Activate(args) + activation.main() + out, err = capsys.readouterr() + assert 'Was unable to find any OSDs to activate' in out + assert 'Verify OSDs are present with ' in out + + def test_detects_running_osds(self, capsys, is_root, capture, monkeypatch): + monkeypatch.setattr('ceph_volume.devices.lvm.activate.direct_report', lambda: direct_report) + monkeypatch.setattr('ceph_volume.devices.lvm.activate.systemctl.osd_is_active', lambda x: True) + args = ['--all'] + activation = activate.Activate(args) + activation.main() + out, err = capsys.readouterr() + assert 'a8789a96ce8b process is active. Skipping activation' in out + assert 'b8218eaa1634 process is active. Skipping activation' in out + + def test_detects_osds_to_activate(self, is_root, capture, monkeypatch): + monkeypatch.setattr('ceph_volume.devices.lvm.activate.direct_report', lambda: direct_report) + monkeypatch.setattr('ceph_volume.devices.lvm.activate.systemctl.osd_is_active', lambda x: False) + args = ['--all'] + activation = activate.Activate(args) + activation.activate = capture + activation.main() + calls = sorted(capture.calls, key=lambda x: x['kwargs']['osd_id']) + assert calls[0]['kwargs']['osd_id'] == '0' + assert calls[0]['kwargs']['osd_fsid'] == '957d22b7-24ce-466a-9883-b8218eaa1634' + assert calls[1]['kwargs']['osd_id'] == '1' + assert calls[1]['kwargs']['osd_fsid'] == 'd0f3e4ad-e52a-4520-afc0-a8789a96ce8b' + +# +# Activate All fixture +# + +direct_report = { + "0": [ + { + "lv_name": "osd-block-957d22b7-24ce-466a-9883-b8218eaa1634", + "lv_path": "/dev/ceph-d4962338-46ff-4cd5-8ea6-c033dbdc5b44/osd-block-957d22b7-24ce-466a-9883-b8218eaa1634", + "lv_tags": "ceph.block_device=/dev/ceph-d4962338-46ff-4cd5-8ea6-c033dbdc5b44/osd-block-957d22b7-24ce-466a-9883-b8218eaa1634,ceph.block_uuid=6MixOd-2Q1I-f8K3-PPOq-UJGV-L3A0-0XwUm4,ceph.cephx_lockbox_secret=,ceph.cluster_fsid=d4962338-46ff-4cd5-8ea6-c033dbdc5b44,ceph.cluster_name=ceph,ceph.crush_device_class=None,ceph.encrypted=0,ceph.osd_fsid=957d22b7-24ce-466a-9883-b8218eaa1634,ceph.osd_id=0,ceph.type=block", + "lv_uuid": "6MixOd-2Q1I-f8K3-PPOq-UJGV-L3A0-0XwUm4", + "name": "osd-block-957d22b7-24ce-466a-9883-b8218eaa1634", + "path": "/dev/ceph-d4962338-46ff-4cd5-8ea6-c033dbdc5b44/osd-block-957d22b7-24ce-466a-9883-b8218eaa1634", + "tags": { + "ceph.block_device": "/dev/ceph-d4962338-46ff-4cd5-8ea6-c033dbdc5b44/osd-block-957d22b7-24ce-466a-9883-b8218eaa1634", + "ceph.block_uuid": "6MixOd-2Q1I-f8K3-PPOq-UJGV-L3A0-0XwUm4", + "ceph.cephx_lockbox_secret": "", + "ceph.cluster_fsid": "d4962338-46ff-4cd5-8ea6-c033dbdc5b44", + "ceph.cluster_name": "ceph", + "ceph.crush_device_class": "None", + "ceph.encrypted": "0", + "ceph.osd_fsid": "957d22b7-24ce-466a-9883-b8218eaa1634", + "ceph.osd_id": "0", + "ceph.type": "block" + }, + "type": "block", + "vg_name": "ceph-d4962338-46ff-4cd5-8ea6-c033dbdc5b44" + } + ], + "1": [ + { + "lv_name": "osd-block-d0f3e4ad-e52a-4520-afc0-a8789a96ce8b", + "lv_path": "/dev/ceph-7538bcf0-f155-4d3f-a9fd-d8b15905e532/osd-block-d0f3e4ad-e52a-4520-afc0-a8789a96ce8b", + "lv_tags": "ceph.block_device=/dev/ceph-7538bcf0-f155-4d3f-a9fd-d8b15905e532/osd-block-d0f3e4ad-e52a-4520-afc0-a8789a96ce8b,ceph.block_uuid=1igwLb-ZlmV-eLgp-hapx-c1Hr-M5gz-sHjnyW,ceph.cephx_lockbox_secret=,ceph.cluster_fsid=d4962338-46ff-4cd5-8ea6-c033dbdc5b44,ceph.cluster_name=ceph,ceph.crush_device_class=None,ceph.encrypted=0,ceph.osd_fsid=d0f3e4ad-e52a-4520-afc0-a8789a96ce8b,ceph.osd_id=1,ceph.type=block", + "lv_uuid": "1igwLb-ZlmV-eLgp-hapx-c1Hr-M5gz-sHjnyW", + "name": "osd-block-d0f3e4ad-e52a-4520-afc0-a8789a96ce8b", + "path": "/dev/ceph-7538bcf0-f155-4d3f-a9fd-d8b15905e532/osd-block-d0f3e4ad-e52a-4520-afc0-a8789a96ce8b", + "tags": { + "ceph.block_device": "/dev/ceph-7538bcf0-f155-4d3f-a9fd-d8b15905e532/osd-block-d0f3e4ad-e52a-4520-afc0-a8789a96ce8b", + "ceph.block_uuid": "1igwLb-ZlmV-eLgp-hapx-c1Hr-M5gz-sHjnyW", + "ceph.cephx_lockbox_secret": "", + "ceph.cluster_fsid": "d4962338-46ff-4cd5-8ea6-c033dbdc5b44", + "ceph.cluster_name": "ceph", + "ceph.crush_device_class": "None", + "ceph.encrypted": "0", + "ceph.osd_fsid": "d0f3e4ad-e52a-4520-afc0-a8789a96ce8b", + "ceph.osd_id": "1", + "ceph.type": "block" + }, + "type": "block", + "vg_name": "ceph-7538bcf0-f155-4d3f-a9fd-d8b15905e532" + } + ] +} diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/Vagrantfile b/ceph/src/ceph-volume/ceph_volume/tests/functional/Vagrantfile index 469ca3390..0f90faf67 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/Vagrantfile +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/Vagrantfile @@ -84,6 +84,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| client.vm.provider :libvirt do |lv| lv.memory = MEMORY lv.random_hostname = true + lv.nic_model_type = "e1000" end # Parallels @@ -122,6 +123,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| rgw.vm.provider :libvirt do |lv| lv.memory = MEMORY lv.random_hostname = true + lv.nic_model_type = "e1000" end # Parallels @@ -160,6 +162,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| nfs.vm.provider :libvirt do |lv| lv.memory = MEMORY lv.random_hostname = true + lv.nic_model_type = "e1000" end # Parallels @@ -197,6 +200,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| mds.vm.provider :libvirt do |lv| lv.memory = MEMORY lv.random_hostname = true + lv.nic_model_type = "e1000" end # Parallels mds.vm.provider "parallels" do |prl| @@ -233,6 +237,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| rbd_mirror.vm.provider :libvirt do |lv| lv.memory = MEMORY lv.random_hostname = true + lv.nic_model_type = "e1000" end # Parallels rbd_mirror.vm.provider "parallels" do |prl| @@ -269,6 +274,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| iscsi_gw.vm.provider :libvirt do |lv| lv.memory = MEMORY lv.random_hostname = true + lv.nic_model_type = "e1000" end # Parallels iscsi_gw.vm.provider "parallels" do |prl| @@ -305,6 +311,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| mon.vm.provider :libvirt do |lv| lv.memory = MEMORY lv.random_hostname = true + lv.nic_model_type = "e1000" end # Parallels @@ -374,6 +381,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| end lv.memory = MEMORY lv.random_hostname = true + lv.nic_model_type = "e1000" end # Parallels diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/dmcrypt/test.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/dmcrypt/test.yml index 2009aaefb..70c018aa7 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/dmcrypt/test.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/dmcrypt/test.yml @@ -8,15 +8,15 @@ name: ceph-osd@2 state: stopped - - name: destroy osd.2 + - name: destroy osd.2 command: "ceph osd destroy osd.2 --yes-i-really-mean-it" - - name: zap /dev/sdd1 + - name: zap /dev/sdd1 command: "ceph-volume lvm zap /dev/sdd1 --destroy" environment: CEPH_VOLUME_DEBUG: 1 - - name: redeploy osd.2 using /dev/sdd1 + - name: redeploy osd.2 using /dev/sdd1 command: "ceph-volume lvm create --bluestore --data /dev/sdd1 --osd-id 2" environment: CEPH_VOLUME_DEBUG: 1 @@ -26,15 +26,38 @@ name: ceph-osd@0 state: stopped - - name: destroy osd.0 + - name: destroy osd.0 command: "ceph osd destroy osd.0 --yes-i-really-mean-it" - - name: zap test_group/data-lv1 + - name: zap test_group/data-lv1 command: "ceph-volume lvm zap test_group/data-lv1" environment: CEPH_VOLUME_DEBUG: 1 - - name: redeploy osd.0 using test_group/data-lv1 + - name: redeploy osd.0 using test_group/data-lv1 command: "ceph-volume lvm create --bluestore --data test_group/data-lv1 --osd-id 0" environment: CEPH_VOLUME_DEBUG: 1 + + - name: stop ceph-osd@0 daemon + service: + name: ceph-osd@0 + state: stopped + + - name: destroy osd.0 + command: "ceph osd destroy osd.0 --yes-i-really-mean-it" + + - name: zap test_group/data-lv1 + command: "ceph-volume lvm zap test_group/data-lv1" + environment: + CEPH_VOLUME_DEBUG: 1 + + - name: prepare osd.0 using test_group/data-lv1 + command: "ceph-volume lvm prepare --bluestore --data test_group/data-lv1 --osd-id 0" + environment: + CEPH_VOLUME_DEBUG: 1 + + - name: activate all to start the previously prepared osd.0 + command: "ceph-volume lvm activate --all" + environment: + CEPH_VOLUME_DEBUG: 1 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/dmcrypt/test.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/dmcrypt/test.yml index c1ade2fbc..fc3e38c7a 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/dmcrypt/test.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/dmcrypt/test.yml @@ -8,20 +8,48 @@ name: ceph-osd@2 state: stopped - - name: destroy osd.2 + - name: destroy osd.2 command: "ceph osd destroy osd.2 --yes-i-really-mean-it" - - name: zap /dev/sdd1 + - name: zap /dev/sdd1 command: "ceph-volume lvm zap /dev/sdd1 --destroy" environment: CEPH_VOLUME_DEBUG: 1 - - name: zap /dev/sdd2 + - name: zap /dev/sdd2 command: "ceph-volume lvm zap /dev/sdd2 --destroy" environment: CEPH_VOLUME_DEBUG: 1 - - name: redeploy osd.2 using /dev/sdd1 + - name: redeploy osd.2 using /dev/sdd1 command: "ceph-volume lvm create --filestore --data /dev/sdd1 --journal /dev/sdd2 --osd-id 2" environment: CEPH_VOLUME_DEBUG: 1 + + - name: stop ceph-osd@0 daemon + service: + name: ceph-osd@0 + state: stopped + + - name: destroy osd.0 + command: "ceph osd destroy osd.0 --yes-i-really-mean-it" + + - name: zap test_group/data-lv1 + command: "ceph-volume lvm zap test_group/data-lv1" + environment: + CEPH_VOLUME_DEBUG: 1 + + - name: zap /dev/sdc1 + command: "ceph-volume lvm zap /dev/sdc1 --destroy" + environment: + CEPH_VOLUME_DEBUG: 1 + + - name: prepare osd.0 again using test_group/data-lv1 + command: "ceph-volume lvm prepare --filestore --data test_group/data-lv1 --journal /dev/sdc1 --osd-id 0" + environment: + CEPH_VOLUME_DEBUG: 1 + + - name: activate all to start the previously prepared osd.0 + command: "ceph-volume lvm activate --filestore --all" + environment: + CEPH_VOLUME_DEBUG: 1 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml index 10bd14e59..ec1e11b83 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml @@ -8,15 +8,38 @@ name: ceph-osd@2 state: stopped - - name: destroy osd.2 + - name: destroy osd.2 command: "ceph osd destroy osd.2 --yes-i-really-mean-it" - - name: zap /dev/sdd1 + - name: zap /dev/sdd1 command: "ceph-volume lvm zap /dev/sdd1 --destroy" environment: CEPH_VOLUME_DEBUG: 1 - - name: redeploy osd.2 using /dev/sdd1 + - name: redeploy osd.2 using /dev/sdd1 command: "ceph-volume lvm create --bluestore --data /dev/sdd1 --osd-id 2" environment: CEPH_VOLUME_DEBUG: 1 + + - name: stop ceph-osd@0 daemon + service: + name: ceph-osd@0 + state: stopped + + - name: destroy osd.0 + command: "ceph osd destroy osd.0 --yes-i-really-mean-it" + + - name: zap test_group/data-lv1 + command: "ceph-volume lvm zap test_group/data-lv1" + environment: + CEPH_VOLUME_DEBUG: 1 + + - name: prepare osd.0 again using test_group/data-lv1 + command: "ceph-volume lvm prepare --bluestore --data test_group/data-lv1 --osd-id 0" + environment: + CEPH_VOLUME_DEBUG: 1 + + - name: activate all to start the previously prepared osd.0 + command: "ceph-volume lvm activate --all" + environment: + CEPH_VOLUME_DEBUG: 1 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml index c1ade2fbc..fc3e38c7a 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_filestore.yml @@ -8,20 +8,48 @@ name: ceph-osd@2 state: stopped - - name: destroy osd.2 + - name: destroy osd.2 command: "ceph osd destroy osd.2 --yes-i-really-mean-it" - - name: zap /dev/sdd1 + - name: zap /dev/sdd1 command: "ceph-volume lvm zap /dev/sdd1 --destroy" environment: CEPH_VOLUME_DEBUG: 1 - - name: zap /dev/sdd2 + - name: zap /dev/sdd2 command: "ceph-volume lvm zap /dev/sdd2 --destroy" environment: CEPH_VOLUME_DEBUG: 1 - - name: redeploy osd.2 using /dev/sdd1 + - name: redeploy osd.2 using /dev/sdd1 command: "ceph-volume lvm create --filestore --data /dev/sdd1 --journal /dev/sdd2 --osd-id 2" environment: CEPH_VOLUME_DEBUG: 1 + + - name: stop ceph-osd@0 daemon + service: + name: ceph-osd@0 + state: stopped + + - name: destroy osd.0 + command: "ceph osd destroy osd.0 --yes-i-really-mean-it" + + - name: zap test_group/data-lv1 + command: "ceph-volume lvm zap test_group/data-lv1" + environment: + CEPH_VOLUME_DEBUG: 1 + + - name: zap /dev/sdc1 + command: "ceph-volume lvm zap /dev/sdc1 --destroy" + environment: + CEPH_VOLUME_DEBUG: 1 + + - name: prepare osd.0 again using test_group/data-lv1 + command: "ceph-volume lvm prepare --filestore --data test_group/data-lv1 --journal /dev/sdc1 --osd-id 0" + environment: + CEPH_VOLUME_DEBUG: 1 + + - name: activate all to start the previously prepared osd.0 + command: "ceph-volume lvm activate --filestore --all" + environment: + CEPH_VOLUME_DEBUG: 1 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini index a2c08e753..429b467f4 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini @@ -10,6 +10,7 @@ whitelist_externals = passenv=* setenv= ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config + ANSIBLE_ACTION_PLUGINS = {envdir}/tmp/ceph-ansible/plugins/actions ANSIBLE_STDOUT_CALLBACK = debug ANSIBLE_RETRY_FILES_ENABLED = False ANSIBLE_SSH_RETRIES = 5 @@ -57,8 +58,8 @@ commands= # test cluster state using ceph-ansible tests testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/tests - # reboot all vms - vagrant reload --no-provision + # reboot all vms - attempt + bash {toxinidir}/../scripts/vagrant_reload.sh {env:VAGRANT_UP_FLAGS:"--no-provision"} {posargs:--provider=virtualbox} # retest to ensure cluster came back up correctly after rebooting testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/tests diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/dmcrypt/test.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/dmcrypt/test.yml index 2009aaefb..9a60ab2e0 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/dmcrypt/test.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/dmcrypt/test.yml @@ -38,3 +38,26 @@ command: "ceph-volume lvm create --bluestore --data test_group/data-lv1 --osd-id 0" environment: CEPH_VOLUME_DEBUG: 1 + + - name: stop ceph-osd@0 daemon + service: + name: ceph-osd@0 + state: stopped + + - name: destroy osd.0 + command: "ceph osd destroy osd.0 --yes-i-really-mean-it" + + - name: zap test_group/data-lv1 + command: "ceph-volume lvm zap test_group/data-lv1" + environment: + CEPH_VOLUME_DEBUG: 1 + + - name: prepare osd.0 using test_group/data-lv1 + command: "ceph-volume lvm prepare --bluestore --data test_group/data-lv1 --osd-id 0" + environment: + CEPH_VOLUME_DEBUG: 1 + + - name: activate all to start the previously prepared osd.0 + command: "ceph-volume lvm activate --all" + environment: + CEPH_VOLUME_DEBUG: 1 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/dmcrypt/test.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/dmcrypt/test.yml index c1ade2fbc..fc3e38c7a 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/dmcrypt/test.yml +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/dmcrypt/test.yml @@ -8,20 +8,48 @@ name: ceph-osd@2 state: stopped - - name: destroy osd.2 + - name: destroy osd.2 command: "ceph osd destroy osd.2 --yes-i-really-mean-it" - - name: zap /dev/sdd1 + - name: zap /dev/sdd1 command: "ceph-volume lvm zap /dev/sdd1 --destroy" environment: CEPH_VOLUME_DEBUG: 1 - - name: zap /dev/sdd2 + - name: zap /dev/sdd2 command: "ceph-volume lvm zap /dev/sdd2 --destroy" environment: CEPH_VOLUME_DEBUG: 1 - - name: redeploy osd.2 using /dev/sdd1 + - name: redeploy osd.2 using /dev/sdd1 command: "ceph-volume lvm create --filestore --data /dev/sdd1 --journal /dev/sdd2 --osd-id 2" environment: CEPH_VOLUME_DEBUG: 1 + + - name: stop ceph-osd@0 daemon + service: + name: ceph-osd@0 + state: stopped + + - name: destroy osd.0 + command: "ceph osd destroy osd.0 --yes-i-really-mean-it" + + - name: zap test_group/data-lv1 + command: "ceph-volume lvm zap test_group/data-lv1" + environment: + CEPH_VOLUME_DEBUG: 1 + + - name: zap /dev/sdc1 + command: "ceph-volume lvm zap /dev/sdc1 --destroy" + environment: + CEPH_VOLUME_DEBUG: 1 + + - name: prepare osd.0 again using test_group/data-lv1 + command: "ceph-volume lvm prepare --filestore --data test_group/data-lv1 --journal /dev/sdc1 --osd-id 0" + environment: + CEPH_VOLUME_DEBUG: 1 + + - name: activate all to start the previously prepared osd.0 + command: "ceph-volume lvm activate --filestore --all" + environment: + CEPH_VOLUME_DEBUG: 1 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/scripts/vagrant_reload.sh b/ceph/src/ceph-volume/ceph_volume/tests/functional/scripts/vagrant_reload.sh new file mode 100644 index 000000000..3211b066d --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/scripts/vagrant_reload.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# vagrant-libvirt has a common behavior where it times out when "reloading" vms. Instead +# of calling `vagrant reload` attempt to halt everything, and then start everything, which gives +# this script the ability to try the `vagrant up` again in case of failure +# + +vagrant halt +# This should not really be needed, but in case of a possible race condition between halt +# and up, it might improve things +sleep 5 + + +retries=0 +until [ $retries -ge 5 ] +do + echo "Attempting to start VMs. Attempts: $retries" + timeout 10m vagrant up "$@" && break + retries=$[$retries+1] + sleep 5 +done diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini index e8e62dc12..55d8c81d0 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini @@ -11,6 +11,7 @@ whitelist_externals = passenv=* setenv= ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config + ANSIBLE_ACTION_PLUGINS = {envdir}/tmp/ceph-ansible/plugins/actions ANSIBLE_STDOUT_CALLBACK = debug ANSIBLE_RETRY_FILES_ENABLED = False ANSIBLE_SSH_RETRIES = 5 @@ -52,7 +53,7 @@ commands= ansible-playbook -vv -i {changedir}/hosts {changedir}/test.yml # reboot all vms - vagrant reload --no-provision + bash {toxinidir}/../scripts/vagrant_reload.sh {env:VAGRANT_UP_FLAGS:"--no-provision"} {posargs:--provider=virtualbox} # wait 2 minutes for services to be ready sleep 120 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/test_process.py b/ceph/src/ceph-volume/ceph_volume/tests/test_process.py new file mode 100644 index 000000000..d38927ae2 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/test_process.py @@ -0,0 +1,68 @@ +import pytest +from ceph_volume.tests.conftest import Factory +from ceph_volume import process + + +@pytest.fixture +def mock_call(monkeypatch): + """ + Monkeypatches process.call, so that a caller can add behavior to the response + """ + def apply(stdout=None, stderr=None, returncode=0): + stdout_stream = Factory(read=lambda: stdout) + stderr_stream = Factory(read=lambda: stderr) + return_value = Factory( + stdout=stdout_stream, + stderr=stderr_stream, + wait=lambda: returncode, + communicate=lambda x: (stdout, stderr, returncode) + ) + + monkeypatch.setattr( + 'ceph_volume.process.subprocess.Popen', + lambda *a, **kw: return_value) + + return apply + + +class TestCall(object): + + def test_stderr_terminal_and_logfile(self, mock_call, caplog, capsys): + mock_call(stdout='stdout\n', stderr='some stderr message\n') + process.call(['ls'], terminal_verbose=True) + out, err = capsys.readouterr() + log_lines = [line[-1] for line in caplog.record_tuples] + assert 'Running command: ' in log_lines[0] + assert 'ls' in log_lines[0] + assert 'stderr some stderr message' in log_lines[-1] + assert 'some stderr message' in out + + def test_stderr_terminal_and_logfile_off(self, mock_call, caplog, capsys): + mock_call(stdout='stdout\n', stderr='some stderr message\n') + process.call(['ls'], terminal_verbose=False) + out, err = capsys.readouterr() + log_lines = [line[-1] for line in caplog.record_tuples] + assert 'Running command: ' in log_lines[0] + assert 'ls' in log_lines[0] + assert 'stderr some stderr message' in log_lines[-1] + assert out == '' + + def test_verbose_on_failure(self, mock_call, caplog, capsys): + mock_call(stdout='stdout\n', stderr='stderr\n', returncode=1) + process.call(['ls'], terminal_verbose=False, logfile_verbose=False) + out, err = capsys.readouterr() + log_lines = '\n'.join([line[-1] for line in caplog.record_tuples]) + assert 'Running command: ' in log_lines + assert 'ls' in log_lines + assert 'stderr' in log_lines + assert 'stdout: stdout' in out + + def test_silent_verbose_on_failure(self, mock_call, caplog, capsys): + mock_call(stdout='stdout\n', stderr='stderr\n', returncode=1) + process.call(['ls'], verbose_on_failure=False) + out, err = capsys.readouterr() + log_lines = '\n'.join([line[-1] for line in caplog.record_tuples]) + assert 'Running command: ' in log_lines + assert 'ls' in log_lines + assert 'stderr' in log_lines + assert out == '' diff --git a/ceph/src/ceph-volume/ceph_volume/tests/util/test_prepare.py b/ceph/src/ceph-volume/ceph_volume/tests/util/test_prepare.py index c65e51f6b..bf825ff46 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/util/test_prepare.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/util/test_prepare.py @@ -235,10 +235,30 @@ class TestNormalizeFlags(object): @pytest.mark.parametrize("flags", ceph_conf_mount_values) def test_normalize_lists(self, flags): - result = prepare._normalize_mount_flags(flags) - assert result == 'rw,auto,exec' + result = sorted(prepare._normalize_mount_flags(flags).split(',')) + assert ','.join(result) == 'auto,exec,rw' @pytest.mark.parametrize("flags", string_mount_values) def test_normalize_strings(self, flags): - result = prepare._normalize_mount_flags(flags) - assert result == 'rw,auto,exec' + result = sorted(prepare._normalize_mount_flags(flags).split(',')) + assert ','.join(result) == 'auto,exec,rw' + + @pytest.mark.parametrize("flags", ceph_conf_mount_values) + def test_normalize_extra_flags(self, flags): + result = prepare._normalize_mount_flags(flags, extras=['discard']) + assert sorted(result.split(',')) == ['auto', 'discard', 'exec', 'rw'] + + @pytest.mark.parametrize("flags", ceph_conf_mount_values) + def test_normalize_duplicate_extra_flags(self, flags): + result = prepare._normalize_mount_flags(flags, extras=['rw', 'discard']) + assert sorted(result.split(',')) == ['auto', 'discard', 'exec', 'rw'] + + @pytest.mark.parametrize("flags", string_mount_values) + def test_normalize_strings_flags(self, flags): + result = sorted(prepare._normalize_mount_flags(flags, extras=['discard']).split(',')) + assert ','.join(result) == 'auto,discard,exec,rw' + + @pytest.mark.parametrize("flags", string_mount_values) + def test_normalize_strings_duplicate_flags(self, flags): + result = sorted(prepare._normalize_mount_flags(flags, extras=['discard','rw']).split(',')) + assert ','.join(result) == 'auto,discard,exec,rw' diff --git a/ceph/src/ceph-volume/ceph_volume/tests/util/test_system.py b/ceph/src/ceph-volume/ceph_volume/tests/util/test_system.py index a742de480..690147cb2 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/util/test_system.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/util/test_system.py @@ -166,3 +166,29 @@ class TestIsBinary(object): def test_is_not_binary(self, tmpfile): binary_path = tmpfile(contents='asd\n\nlkjh0') assert system.is_binary(binary_path) is False + + +class TestWhich(object): + + def test_executable_exists_but_is_not_file(self, monkeypatch): + monkeypatch.setattr(system.os.path, 'isfile', lambda x: False) + monkeypatch.setattr(system.os.path, 'exists', lambda x: True) + assert system.which('exedir') == 'exedir' + + def test_executable_does_not_exist(self, monkeypatch): + monkeypatch.setattr(system.os.path, 'isfile', lambda x: False) + monkeypatch.setattr(system.os.path, 'exists', lambda x: False) + assert system.which('exedir') == 'exedir' + + def test_executable_exists_as_file(self, monkeypatch): + monkeypatch.setattr(system.os.path, 'isfile', lambda x: True) + monkeypatch.setattr(system.os.path, 'exists', lambda x: True) + assert system.which('ceph') == '/usr/local/bin/ceph' + + def test_warnings_when_executable_isnt_matched(self, monkeypatch, capsys): + monkeypatch.setattr(system.os.path, 'isfile', lambda x: True) + monkeypatch.setattr(system.os.path, 'exists', lambda x: False) + system.which('exedir') + stdout, stderr = capsys.readouterr() + assert 'Absolute path not found for executable: exedir' in stdout + assert 'Ensure $PATH environment variable contains common executable locations' in stdout diff --git a/ceph/src/ceph-volume/ceph_volume/util/disk.py b/ceph/src/ceph-volume/ceph_volume/util/disk.py index c3a791540..434723340 100644 --- a/ceph/src/ceph-volume/ceph_volume/util/disk.py +++ b/ceph/src/ceph-volume/ceph_volume/util/disk.py @@ -179,18 +179,6 @@ def lsblk(device, columns=None, abspath=False): return _lsblk_parser(' '.join(out)) -def _lsblk_type(device): - """ - Helper function that will use the ``TYPE`` label output of ``lsblk`` to determine - if a device is a partition or disk. - It does not process the output to return a boolean, but it does process it to return the - """ - out, err, rc = process.call( - ['blkid', '-s', 'PARTUUID', '-o', 'value', device] - ) - return ' '.join(out).strip() - - def is_device(dev): """ Boolean to determine if a given device is a block device (**not** diff --git a/ceph/src/ceph-volume/ceph_volume/util/prepare.py b/ceph/src/ceph-volume/ceph_volume/util/prepare.py index 3e1a77f90..d02c570fe 100644 --- a/ceph/src/ceph-volume/ceph_volume/util/prepare.py +++ b/ceph/src/ceph-volume/ceph_volume/util/prepare.py @@ -144,7 +144,7 @@ def format_device(device): process.run(command) -def _normalize_mount_flags(flags): +def _normalize_mount_flags(flags, extras=None): """ Mount flag options have to be a single string, separated by a comma. If the flags are separated by spaces, or with commas and spaces in ceph.conf, the @@ -159,20 +159,45 @@ def _normalize_mount_flags(flags): [" rw ,", "exec"] :param flags: A list of flags, or a single string of mount flags + :param extras: Extra set of mount flags, useful when custom devices like VDO need + ad-hoc mount configurations """ + # Instead of using set(), we append to this new list here, because set() + # will create an arbitrary order on the items that is made worst when + # testing with tools like tox that includes a randomizer seed. By + # controlling the order, it is easier to correctly assert the expectation + unique_flags = [] if isinstance(flags, list): + if extras: + flags.extend(extras) + # ensure that spaces and commas are removed so that they can join - # correctly - return ','.join([f.strip().strip(',') for f in flags if f]) + # correctly, remove duplicates + for f in flags: + if f and f not in unique_flags: + unique_flags.append(f.strip().strip(',')) + return ','.join(unique_flags) # split them, clean them, and join them back again flags = flags.strip().split(' ') - return ','.join( - [f.strip().strip(',') for f in flags if f] - ) - - -def mount_osd(device, osd_id): + if extras: + flags.extend(extras) + + # remove possible duplicates + for f in flags: + if f and f not in unique_flags: + unique_flags.append(f.strip().strip(',')) + flags = ','.join(unique_flags) + # Before returning, split them again, since strings can be mashed up + # together, preventing removal of duplicate entries + return ','.join(set(flags.split(','))) + + +def mount_osd(device, osd_id, **kw): + extras = [] + is_vdo = kw.get('is_vdo', '0') + if is_vdo == '1': + extras = ['discard'] destination = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id) command = ['mount', '-t', 'xfs', '-o'] flags = conf.ceph.get_list( @@ -181,7 +206,9 @@ def mount_osd(device, osd_id): default=constants.mount.get('xfs'), split=' ', ) - command.append(_normalize_mount_flags(flags)) + command.append( + _normalize_mount_flags(flags, extras=extras) + ) command.append(device) command.append(destination) process.run(command) diff --git a/ceph/src/ceph-volume/ceph_volume/util/system.py b/ceph/src/ceph-volume/ceph_volume/util/system.py index 2b4473efe..0ba46d362 100644 --- a/ceph/src/ceph-volume/ceph_volume/util/system.py +++ b/ceph/src/ceph-volume/ceph_volume/util/system.py @@ -5,10 +5,11 @@ import pwd import platform import tempfile import uuid -from ceph_volume import process +from ceph_volume import process, terminal from . import as_string logger = logging.getLogger(__name__) +mlogger = terminal.MultiLogger(__name__) # TODO: get these out of here and into a common area for others to consume if platform.system() == 'FreeBSD': @@ -30,6 +31,28 @@ def generate_uuid(): return str(uuid.uuid4()) +def which(executable): + """find the location of an executable""" + locations = ( + '/usr/local/bin', + '/bin', + '/usr/bin', + '/usr/local/sbin', + '/usr/sbin', + '/sbin', + ) + + for location in locations: + executable_path = os.path.join(location, executable) + if os.path.exists(executable_path) and os.path.isfile(executable_path): + return executable_path + mlogger.warning('Absolute path not found for executable: %s', executable) + mlogger.warning('Ensure $PATH environment variable contains common executable locations') + # fallback to just returning the argument as-is, to prevent a hard fail, + # and hoping that the system might have the executable somewhere custom + return executable + + def get_ceph_user_ids(): """ Return the id and gid of the ceph user diff --git a/ceph/src/client/Client.cc b/ceph/src/client/Client.cc index e0b077189..1fd435839 100644 --- a/ceph/src/client/Client.cc +++ b/ceph/src/client/Client.cc @@ -3408,8 +3408,10 @@ void Client::check_caps(Inode *in, unsigned flags) return; // guard if at end of func if ((revoking & (CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO)) && - (used & CEPH_CAP_FILE_CACHE) && !(used & CEPH_CAP_FILE_BUFFER)) - _release(in); + (used & CEPH_CAP_FILE_CACHE) && !(used & CEPH_CAP_FILE_BUFFER)) { + if (_release(in)) + used &= ~CEPH_CAP_FILE_CACHE; + } if (!in->cap_snaps.empty()) flush_snaps(in); @@ -3743,8 +3745,11 @@ void Client::_invalidate_inode_cache(Inode *in) ldout(cct, 10) << "_invalidate_inode_cache " << *in << dendl; // invalidate our userspace inode cache - if (cct->_conf->client_oc) + if (cct->_conf->client_oc) { objectcacher->release_set(&in->oset); + if (!objectcacher->set_is_empty(&in->oset)) + lderr(cct) << "failed to invalidate cache for " << *in << dendl; + } _schedule_invalidate_callback(in, 0, 0); } @@ -4064,12 +4069,14 @@ void Client::_invalidate_kernel_dcache() { if (unmounting) return; - if (can_invalidate_dentries && dentry_invalidate_cb && root->dir) { - for (ceph::unordered_map::iterator p = root->dir->dentries.begin(); - p != root->dir->dentries.end(); - ++p) { - if (p->second->inode) - _schedule_invalidate_dentry_callback(p->second, false); + if (can_invalidate_dentries) { + if (dentry_invalidate_cb && root->dir) { + for (ceph::unordered_map::iterator p = root->dir->dentries.begin(); + p != root->dir->dentries.end(); + ++p) { + if (p->second->inode) + _schedule_invalidate_dentry_callback(p->second, false); + } } } else if (remount_cb) { // Hacky: @@ -6151,7 +6158,7 @@ int Client::_lookup(Inode *dir, const string& dname, int mask, InodeRef *target, << " seq " << dn->lease_seq << dendl; - if (!dn->inode || dn->inode->caps_issued_mask(mask)) { + if (!dn->inode || dn->inode->caps_issued_mask(mask, true)) { // is dn lease valid? utime_t now = ceph_clock_now(); if (dn->lease_mds >= 0 && @@ -6169,9 +6176,9 @@ int Client::_lookup(Inode *dir, const string& dname, int mask, InodeRef *target, << " vs lease_gen " << dn->lease_gen << dendl; } // dir lease? - if (dir->caps_issued_mask(CEPH_CAP_FILE_SHARED)) { + if (dir->caps_issued_mask(CEPH_CAP_FILE_SHARED, true)) { if (dn->cap_shared_gen == dir->shared_gen && - (!dn->inode || dn->inode->caps_issued_mask(mask))) + (!dn->inode || dn->inode->caps_issued_mask(mask, true))) goto hit_dn; if (!dn->inode && (dir->flags & I_COMPLETE)) { ldout(cct, 10) << "_lookup concluded ENOENT locally for " @@ -6184,7 +6191,7 @@ int Client::_lookup(Inode *dir, const string& dname, int mask, InodeRef *target, } } else { // can we conclude ENOENT locally? - if (dir->caps_issued_mask(CEPH_CAP_FILE_SHARED) && + if (dir->caps_issued_mask(CEPH_CAP_FILE_SHARED, true) && (dir->flags & I_COMPLETE)) { ldout(cct, 10) << "_lookup concluded ENOENT locally for " << *dir << " dn '" << dname << "'" << dendl; return -ENOENT; @@ -6648,7 +6655,7 @@ int Client::_readlink(Inode *in, char *buf, size_t size) int Client::_getattr(Inode *in, int mask, const UserPerm& perms, bool force) { - bool yes = in->caps_issued_mask(mask); + bool yes = in->caps_issued_mask(mask, true); ldout(cct, 10) << "_getattr mask " << ccap_string(mask) << " issued=" << yes << dendl; if (yes && !force) @@ -6828,7 +6835,7 @@ force_request: } if (mask & CEPH_SETATTR_MTIME) { req->head.args.setattr.mtime = utime_t(stx->stx_mtime); - req->inode_drop |= CEPH_CAP_AUTH_SHARED | CEPH_CAP_FILE_RD | + req->inode_drop |= CEPH_CAP_FILE_SHARED | CEPH_CAP_FILE_RD | CEPH_CAP_FILE_WR; } if (mask & CEPH_SETATTR_ATIME) { @@ -6845,7 +6852,7 @@ force_request: ldout(cct,10) << "unable to set size to " << stx->stx_size << ". Too large!" << dendl; return -EFBIG; } - req->inode_drop |= CEPH_CAP_AUTH_SHARED | CEPH_CAP_FILE_RD | + req->inode_drop |= CEPH_CAP_FILE_SHARED | CEPH_CAP_FILE_RD | CEPH_CAP_FILE_WR; } req->head.args.setattr.mask = mask; @@ -7642,7 +7649,7 @@ int Client::_readdir_get_frag(dir_result_t *dirp) req->head.args.readdir.frag = fg; req->head.args.readdir.flags = CEPH_READDIR_REPLY_BITFLAGS; if (dirp->last_name.length()) { - req->path2.set_path(dirp->last_name.c_str()); + req->path2.set_path(dirp->last_name); } else if (dirp->hash_order()) { req->head.args.readdir.offset_hash = dirp->offset_high(); } @@ -7819,10 +7826,10 @@ int Client::readdir_r_cb(dir_result_t *d, add_dirent_cb_t cb, void *p, if (diri->dn_set.empty()) in = diri; else - in = diri->get_first_parent()->inode; + in = diri->get_first_parent()->dir->parent_inode; int r; - r = _getattr(diri, caps, dirp->perms); + r = _getattr(in, caps, dirp->perms); if (r < 0) return r; @@ -7854,7 +7861,7 @@ int Client::readdir_r_cb(dir_result_t *d, add_dirent_cb_t cb, void *p, << dendl; if (dirp->inode->snapid != CEPH_SNAPDIR && dirp->inode->is_complete_and_ordered() && - dirp->inode->caps_issued_mask(CEPH_CAP_FILE_SHARED)) { + dirp->inode->caps_issued_mask(CEPH_CAP_FILE_SHARED, true)) { int err = _readdir_cache_cb(dirp, cb, p, caps, getref); if (err != -EAGAIN) return err; @@ -9553,7 +9560,7 @@ int Client::fstatx(int fd, struct ceph_statx *stx, const UserPerm& perms, unsigned mask = statx_to_mask(flags, want); int r = 0; - if (mask && !f->inode->caps_issued_mask(mask)) { + if (mask && !f->inode->caps_issued_mask(mask, true)) { r = _getattr(f->inode, mask, perms); if (r < 0) { ldout(cct, 3) << "fstatx exit on error!" << dendl; @@ -10571,7 +10578,7 @@ int Client::ll_getattrx(Inode *in, struct ceph_statx *stx, unsigned int want, int res = 0; unsigned mask = statx_to_mask(flags, want); - if (mask && !in->caps_issued_mask(mask)) + if (mask && !in->caps_issued_mask(mask, true)) res = _ll_getattr(in, mask, perms); if (res == 0) diff --git a/ceph/src/client/Inode.cc b/ceph/src/client/Inode.cc index d29fb25d3..2fadb2b5b 100644 --- a/ceph/src/client/Inode.cc +++ b/ceph/src/client/Inode.cc @@ -198,6 +198,13 @@ int Inode::caps_issued(int *implemented) const c |= it->second->issued; i |= it->second->implemented; } + + // exclude caps issued by non-auth MDS, but are been revoking by + // the auth MDS. The non-auth MDS should be revoking/exporting + // these caps, but the message is delayed. + if (auth_cap) + c &= ~auth_cap->implemented | auth_cap->issued; + if (implemented) *implemented = i; return c; @@ -215,9 +222,27 @@ void Inode::try_touch_cap(mds_rank_t mds) touch_cap(caps[mds]); } -bool Inode::caps_issued_mask(unsigned mask) +/** + * caps_issued_mask - check whether we have all of the caps in the mask + * @mask: mask to check against + * @allow_impl: whether the caller can also use caps that are implemented but not issued + * + * This is the bog standard "check whether we have the required caps" operation. + * Typically, we only check against the capset that is currently "issued". + * In other words, we ignore caps that have been revoked but not yet released. + * + * Some callers (particularly those doing attribute retrieval) can also make + * use of the full set of "implemented" caps to satisfy requests from the + * cache. + * + * Those callers should refrain from taking new references to implemented + * caps! + */ +bool Inode::caps_issued_mask(unsigned mask, bool allow_impl) { int c = snap_caps; + int i = 0; + if ((c & mask) == mask) return true; // prefer auth cap @@ -237,8 +262,13 @@ bool Inode::caps_issued_mask(unsigned mask) return true; } c |= it->second->issued; + i |= it->second->implemented; } } + + if (allow_impl) + c |= i; + if ((c & mask) == mask) { // bah.. touch them all for (map::iterator it = caps.begin(); diff --git a/ceph/src/client/Inode.h b/ceph/src/client/Inode.h index 71586cad7..5242630d5 100644 --- a/ceph/src/client/Inode.h +++ b/ceph/src/client/Inode.h @@ -283,7 +283,7 @@ struct Inode { int caps_issued(int *implemented = 0) const; void touch_cap(Cap *cap); void try_touch_cap(mds_rank_t mds); - bool caps_issued_mask(unsigned mask); + bool caps_issued_mask(unsigned mask, bool allow_impl=false); int caps_used(); int caps_file_wanted(); int caps_wanted(); diff --git a/ceph/src/client/MetaRequest.h b/ceph/src/client/MetaRequest.h index 8880933d7..56dc4d973 100644 --- a/ceph/src/client/MetaRequest.h +++ b/ceph/src/client/MetaRequest.h @@ -169,7 +169,7 @@ public: void set_retry_attempt(int a) { head.num_retry = a; } void set_filepath(const filepath& fp) { path = fp; } void set_filepath2(const filepath& fp) { path2 = fp; } - void set_string2(const char *s) { path2.set_path(s, 0); } + void set_string2(const char *s) { path2.set_path(boost::string_view(s), 0); } void set_caller_perms(const UserPerm& _perms) { perms.shallow_copy(_perms); head.caller_uid = perms.uid(); diff --git a/ceph/src/client/SyntheticClient.cc b/ceph/src/client/SyntheticClient.cc index 88574e07a..ffa11933c 100644 --- a/ceph/src/client/SyntheticClient.cc +++ b/ceph/src/client/SyntheticClient.cc @@ -3358,7 +3358,7 @@ int SyntheticClient::chunk_file(string &filename) uint64_t size = st.st_size; dout(0) << "file " << filename << " size is " << size << dendl; - inode_t inode; + inode_t<> inode; memset(&inode, 0, sizeof(inode)); inode.ino = st.st_ino; ret = client->fdescribe_layout(fd, &inode.layout); diff --git a/ceph/src/cls/rbd/cls_rbd.cc b/ceph/src/cls/rbd/cls_rbd.cc index b37488fb9..b8afbe50e 100644 --- a/ceph/src/cls/rbd/cls_rbd.cc +++ b/ceph/src/cls/rbd/cls_rbd.cc @@ -5474,7 +5474,7 @@ CLS_INIT(rbd) CLS_METHOD_RD | CLS_METHOD_WR, group_image_remove, &h_group_image_remove); cls_register_cxx_method(h_class, "group_image_list", - CLS_METHOD_RD | CLS_METHOD_WR, + CLS_METHOD_RD, group_image_list, &h_group_image_list); cls_register_cxx_method(h_class, "group_image_set", CLS_METHOD_RD | CLS_METHOD_WR, diff --git a/ceph/src/cls/rgw/cls_rgw.cc b/ceph/src/cls/rgw/cls_rgw.cc index 564e81d17..dc9da5968 100644 --- a/ceph/src/cls/rgw/cls_rgw.cc +++ b/ceph/src/cls/rgw/cls_rgw.cc @@ -1684,7 +1684,7 @@ static int rgw_bucket_unlink_instance(cls_method_context_t hctx, bufferlist *in, rgw_bucket_entry_ver ver; ver.epoch = (op.olh_epoch ? op.olh_epoch : olh.get_epoch()); - real_time mtime = real_clock::now(); /* mtime has no real meaning in instance removal context */ + real_time mtime = obj.mtime(); /* mtime has no real meaning in instance removal context */ ret = log_index_operation(hctx, op.key, CLS_RGW_OP_UNLINK_INSTANCE, op.op_tag, mtime, ver, CLS_RGW_STATE_COMPLETE, header.ver, header.max_marker, @@ -2611,6 +2611,9 @@ static int bi_log_iterate_entries(cls_method_context_t hctx, const string& marke if (key.compare(end_key) > 0) { key_iter = key; + if (truncated) { + *truncated = false; + } return 0; } @@ -2917,6 +2920,8 @@ static int usage_iterate_range(cls_method_context_t hctx, uint64_t start, uint64 string user_key; bool truncated_status = false; + assert(truncated != nullptr); + if (!by_user) { usage_record_prefix_by_time(end, end_key); } else { @@ -2939,9 +2944,7 @@ static int usage_iterate_range(cls_method_context_t hctx, uint64_t start, uint64 if (ret < 0) return ret; - if (truncated) { - *truncated = truncated_status; - } + *truncated = truncated_status; map::iterator iter = keys.begin(); if (iter == keys.end()) @@ -2955,12 +2958,14 @@ static int usage_iterate_range(cls_method_context_t hctx, uint64_t start, uint64 if (!by_user && key.compare(end_key) >= 0) { CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str()); + *truncated = false; key_iter = key; return 0; } if (by_user && key.compare(0, user_key.size(), user_key) != 0) { CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str()); + *truncated = false; key_iter = key; return 0; } @@ -2973,8 +2978,10 @@ static int usage_iterate_range(cls_method_context_t hctx, uint64_t start, uint64 continue; /* keys are sorted by epoch, so once we're past end we're done */ - if (e.epoch >= end) + if (e.epoch >= end) { + *truncated = false; return 0; + } ret = cb(hctx, key, e, param); if (ret < 0) @@ -3037,6 +3044,10 @@ int rgw_user_usage_log_read(cls_method_context_t hctx, bufferlist *in, bufferlis static int usage_log_trim_cb(cls_method_context_t hctx, const string& key, rgw_usage_log_entry& entry, void *param) { + bool *found = (bool *)param; + if (found) { + *found = true; + } string key_by_time; string key_by_user; @@ -3072,12 +3083,13 @@ int rgw_user_usage_log_trim(cls_method_context_t hctx, bufferlist *in, bufferlis string iter; bool more; + bool found = false; #define MAX_USAGE_TRIM_ENTRIES 128 - ret = usage_iterate_range(hctx, op.start_epoch, op.end_epoch, op.user, iter, MAX_USAGE_TRIM_ENTRIES, &more, usage_log_trim_cb, NULL); + ret = usage_iterate_range(hctx, op.start_epoch, op.end_epoch, op.user, iter, MAX_USAGE_TRIM_ENTRIES, &more, usage_log_trim_cb, (void *)&found); if (ret < 0) return ret; - if (!more && iter.empty()) + if (!more && !found) return -ENODATA; return 0; diff --git a/ceph/src/cls/user/cls_user.cc b/ceph/src/cls/user/cls_user.cc index 840470e9f..9a9c5bd9f 100644 --- a/ceph/src/cls/user/cls_user.cc +++ b/ceph/src/cls/user/cls_user.cc @@ -359,6 +359,52 @@ static int cls_user_get_header(cls_method_context_t hctx, bufferlist *in, buffer return 0; } +/// A method to reset the user.buckets header stats in accordance to the values +/// seen in the user.buckets omap keys. This will not be equivalent to --sync-stats +/// which requires comparing the values with actual bucket meta stats supplied +/// by RGW +static int cls_user_reset_stats(cls_method_context_t hctx, bufferlist *in, bufferlist *out /*ignore*/) +{ + cls_user_reset_stats_op op; + + try { + auto bliter = in->begin(); + ::decode(op, bliter); + } catch (buffer::error& err) { + CLS_LOG(0, "ERROR: cls_user_reset_op(): failed to decode op"); + return -EINVAL; + } + cls_user_header header; + bool truncated = false; + string from_index, prefix; + do { + map keys; + int rc = cls_cxx_map_get_vals(hctx, from_index, prefix, MAX_ENTRIES, &keys, &truncated); + + if (rc < 0) + return rc; + + for (const auto&kv : keys){ + cls_user_bucket_entry e; + try { + auto bl = kv.second; + auto bliter = bl.begin(); + decode(e, bliter); + } catch (buffer::error& err) { + CLS_LOG(0, "ERROR: failed to decode bucket entry for %s", kv.first.c_str()); + return -EIO; + } + add_header_stats(&header.stats, e); + } + } while (truncated); + + bufferlist bl; + header.last_stats_update = op.time; + ::encode(header, bl); + + return cls_cxx_map_write_header(hctx, &bl); +} + CLS_INIT(user) { CLS_LOG(1, "Loaded user class!"); @@ -369,6 +415,7 @@ CLS_INIT(user) cls_method_handle_t h_user_remove_bucket; cls_method_handle_t h_user_list_buckets; cls_method_handle_t h_user_get_header; + cls_method_handle_t h_user_reset_stats; cls_register("user", &h_class); @@ -380,7 +427,7 @@ CLS_INIT(user) cls_register_cxx_method(h_class, "remove_bucket", CLS_METHOD_RD | CLS_METHOD_WR, cls_user_remove_bucket, &h_user_remove_bucket); cls_register_cxx_method(h_class, "list_buckets", CLS_METHOD_RD, cls_user_list_buckets, &h_user_list_buckets); cls_register_cxx_method(h_class, "get_header", CLS_METHOD_RD, cls_user_get_header, &h_user_get_header); - + cls_register_cxx_method(h_class, "reset_user_stats", CLS_METHOD_RD | CLS_METHOD_WR, cls_user_reset_stats, &h_user_reset_stats); return; } diff --git a/ceph/src/cls/user/cls_user_client.cc b/ceph/src/cls/user/cls_user_client.cc index c77075dcb..5e0d65a8d 100644 --- a/ceph/src/cls/user/cls_user_client.cc +++ b/ceph/src/cls/user/cls_user_client.cc @@ -132,6 +132,15 @@ void cls_user_get_header(librados::ObjectReadOperation& op, op.exec("user", "get_header", inbl, new ClsUserGetHeaderCtx(header, NULL, pret)); } +void cls_user_reset_stats(librados::ObjectWriteOperation &op) +{ + bufferlist inbl; + cls_user_reset_stats_op call; + call.time = real_clock::now(); + ::encode(call, inbl); + op.exec("user", "reset_user_stats", inbl); +} + int cls_user_get_header_async(IoCtx& io_ctx, string& oid, RGWGetUserHeader_CB *ctx) { bufferlist in, out; diff --git a/ceph/src/cls/user/cls_user_client.h b/ceph/src/cls/user/cls_user_client.h index 5e5d3063d..b398961e0 100644 --- a/ceph/src/cls/user/cls_user_client.h +++ b/ceph/src/cls/user/cls_user_client.h @@ -36,5 +36,6 @@ void cls_user_bucket_list(librados::ObjectReadOperation& op, int *pret); void cls_user_get_header(librados::ObjectReadOperation& op, cls_user_header *header, int *pret); int cls_user_get_header_async(librados::IoCtx& io_ctx, string& oid, RGWGetUserHeader_CB *ctx); +void cls_user_reset_stats(librados::ObjectWriteOperation& op); #endif diff --git a/ceph/src/cls/user/cls_user_ops.h b/ceph/src/cls/user/cls_user_ops.h index 2219a4959..a79e03cc6 100644 --- a/ceph/src/cls/user/cls_user_ops.h +++ b/ceph/src/cls/user/cls_user_ops.h @@ -135,6 +135,27 @@ struct cls_user_get_header_op { }; WRITE_CLASS_ENCODER(cls_user_get_header_op) +struct cls_user_reset_stats_op { + real_time time; + cls_user_reset_stats_op() {} + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(time, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(time, bl); + DECODE_FINISH(bl); + } + + void dump(Formatter *f) const; + static void generate_test_instances(list& ls); +}; +WRITE_CLASS_ENCODER(cls_user_reset_stats_op); + struct cls_user_get_header_ret { cls_user_header header; diff --git a/ceph/src/common/Formatter.cc b/ceph/src/common/Formatter.cc index 3b3589db8..d39a96f5a 100644 --- a/ceph/src/common/Formatter.cc +++ b/ceph/src/common/Formatter.cc @@ -52,13 +52,13 @@ Formatter::Formatter() { } Formatter::~Formatter() { } -Formatter *Formatter::create(const std::string &type, - const std::string& default_type, - const std::string& fallback) +Formatter *Formatter::create(boost::string_view type, + boost::string_view default_type, + boost::string_view fallback) { - std::string mytype = type; + std::string mytype(type); if (mytype == "") - mytype = default_type; + mytype = std::string(default_type); if (mytype == "json") return new JSONFormatter(false); @@ -161,11 +161,11 @@ void JSONFormatter::print_comma(json_formatter_stack_entry_d& entry) m_ss << " "; } -void JSONFormatter::print_quoted_string(const std::string& s) +void JSONFormatter::print_quoted_string(boost::string_view s) { - int len = escape_json_attr_len(s.c_str(), s.size()); + int len = escape_json_attr_len(s.data(), s.size()); char escaped[len]; - escape_json_attr(s.c_str(), s.size(), escaped); + escape_json_attr(s.data(), s.size(), escaped); m_ss << '\"' << escaped << '\"'; } @@ -272,7 +272,7 @@ void JSONFormatter::dump_float(const char *name, double d) m_ss << foo; } -void JSONFormatter::dump_string(const char *name, const std::string& s) +void JSONFormatter::dump_string(const char *name, boost::string_view s) { print_name(name); print_quoted_string(s); @@ -443,19 +443,19 @@ void XMLFormatter::dump_float(const char *name, double d) m_ss << "\n"; } -void XMLFormatter::dump_string(const char *name, const std::string& s) +void XMLFormatter::dump_string(const char *name, boost::string_view s) { std::string e(name); std::transform(e.begin(), e.end(), e.begin(), [this](char c) { return this->to_lower_underscore(c); }); print_spaces(); - m_ss << "<" << e << ">" << escape_xml_str(s.c_str()) << ""; + m_ss << "<" << e << ">" << escape_xml_str(s.data()) << ""; if (m_pretty) m_ss << "\n"; } -void XMLFormatter::dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs) +void XMLFormatter::dump_string_with_attrs(const char *name, boost::string_view s, const FormatterAttrs& attrs) { std::string e(name); std::transform(e.begin(), e.end(), e.begin(), @@ -464,7 +464,7 @@ void XMLFormatter::dump_string_with_attrs(const char *name, const std::string& s std::string attrs_str; get_attrs_str(&attrs, attrs_str); print_spaces(); - m_ss << "<" << e << attrs_str << ">" << escape_xml_str(s.c_str()) << ""; + m_ss << "<" << e << attrs_str << ">" << escape_xml_str(s.data()) << ""; if (m_pretty) m_ss << "\n"; } @@ -564,11 +564,11 @@ void XMLFormatter::print_spaces() } } -std::string XMLFormatter::escape_xml_str(const char *str) +std::string XMLFormatter::escape_xml_str(boost::string_view str) { - int len = escape_xml_attr_len(str); + size_t len = escape_xml_attr_len(str.data()); std::vector escaped(len, '\0'); - escape_xml_attr(str, &escaped[0]); + escape_xml_attr(str.data(), &escaped[0]); return std::string(&escaped[0]); } @@ -837,7 +837,7 @@ void TableFormatter::dump_float(const char *name, double d) m_ss.str(""); } -void TableFormatter::dump_string(const char *name, const std::string& s) +void TableFormatter::dump_string(const char *name, boost::string_view s) { finish_pending_string(); size_t i = m_vec_index(name); @@ -848,7 +848,7 @@ void TableFormatter::dump_string(const char *name, const std::string& s) m_ss.str(""); } -void TableFormatter::dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs) +void TableFormatter::dump_string_with_attrs(const char *name, boost::string_view s, const FormatterAttrs& attrs) { finish_pending_string(); size_t i = m_vec_index(name); diff --git a/ceph/src/common/Formatter.h b/ceph/src/common/Formatter.h index df6c0a99a..14d5f41ca 100644 --- a/ceph/src/common/Formatter.h +++ b/ceph/src/common/Formatter.h @@ -6,6 +6,8 @@ #include "include/int_types.h" #include "include/buffer_fwd.h" +#include + #include #include #include @@ -23,14 +25,14 @@ namespace ceph { class Formatter { public: - static Formatter *create(const std::string& type, - const std::string& default_type, - const std::string& fallback); - static Formatter *create(const std::string& type, - const std::string& default_type) { + static Formatter *create(boost::string_view type, + boost::string_view default_type, + boost::string_view fallback); + static Formatter *create(boost::string_view type, + boost::string_view default_type) { return create(type, default_type, ""); } - static Formatter *create(const std::string& type) { + static Formatter *create(boost::string_view type) { return create(type, "json-pretty", ""); } @@ -54,7 +56,7 @@ namespace ceph { virtual void dump_unsigned(const char *name, uint64_t u) = 0; virtual void dump_int(const char *name, int64_t s) = 0; virtual void dump_float(const char *name, double d) = 0; - virtual void dump_string(const char *name, const std::string& s) = 0; + virtual void dump_string(const char *name, boost::string_view s) = 0; virtual void dump_bool(const char *name, bool b) { dump_format_unquoted(name, "%s", (b ? "true" : "false")); @@ -81,7 +83,7 @@ namespace ceph { { open_object_section(name); } - virtual void dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs) + virtual void dump_string_with_attrs(const char *name, boost::string_view s, const FormatterAttrs& attrs) { dump_string(name, s); } @@ -106,7 +108,7 @@ namespace ceph { void dump_unsigned(const char *name, uint64_t u) override; void dump_int(const char *name, int64_t u) override; void dump_float(const char *name, double d) override; - void dump_string(const char *name, const std::string& s) override; + void dump_string(const char *name, boost::string_view s) override; std::ostream& dump_stream(const char *name) override; void dump_format_va(const char *name, const char *ns, bool quoted, const char *fmt, va_list ap) override; int get_len() const override; @@ -122,7 +124,7 @@ namespace ceph { bool m_pretty; void open_section(const char *name, bool is_array); - void print_quoted_string(const std::string& s); + void print_quoted_string(boost::string_view s); void print_name(const char *name); void print_comma(json_formatter_stack_entry_d& entry); void finish_pending_string(); @@ -154,7 +156,7 @@ namespace ceph { void dump_unsigned(const char *name, uint64_t u) override; void dump_int(const char *name, int64_t u) override; void dump_float(const char *name, double d) override; - void dump_string(const char *name, const std::string& s) override; + void dump_string(const char *name, boost::string_view s) override; std::ostream& dump_stream(const char *name) override; void dump_format_va(const char *name, const char *ns, bool quoted, const char *fmt, va_list ap) override; int get_len() const override; @@ -163,13 +165,13 @@ namespace ceph { /* with attrs */ void open_array_section_with_attrs(const char *name, const FormatterAttrs& attrs) override; void open_object_section_with_attrs(const char *name, const FormatterAttrs& attrs) override; - void dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs) override; + void dump_string_with_attrs(const char *name, boost::string_view s, const FormatterAttrs& attrs) override; protected: void open_section_in_ns(const char *name, const char *ns, const FormatterAttrs *attrs); void finish_pending_string(); void print_spaces(); - static std::string escape_xml_str(const char *str); + static std::string escape_xml_str(boost::string_view str); void get_attrs_str(const FormatterAttrs *attrs, std::string& attrs_str); char to_lower_underscore(char c) const; @@ -206,9 +208,9 @@ namespace ceph { void dump_unsigned(const char *name, uint64_t u) override; void dump_int(const char *name, int64_t u) override; void dump_float(const char *name, double d) override; - void dump_string(const char *name, const std::string& s) override; + void dump_string(const char *name, boost::string_view s) override; void dump_format_va(const char *name, const char *ns, bool quoted, const char *fmt, va_list ap) override; - void dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs) override; + void dump_string_with_attrs(const char *name, boost::string_view s, const FormatterAttrs& attrs) override; std::ostream& dump_stream(const char *name) override; int get_len() const override; diff --git a/ceph/src/common/HTMLFormatter.cc b/ceph/src/common/HTMLFormatter.cc index 02ef744c4..b9344b74b 100644 --- a/ceph/src/common/HTMLFormatter.cc +++ b/ceph/src/common/HTMLFormatter.cc @@ -107,18 +107,18 @@ void HTMLFormatter::dump_float(const char *name, double d) dump_template(name, d); } -void HTMLFormatter::dump_string(const char *name, const std::string& s) +void HTMLFormatter::dump_string(const char *name, boost::string_view s) { - dump_template(name, escape_xml_str(s.c_str())); + dump_template(name, escape_xml_str(s.data())); } -void HTMLFormatter::dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs) +void HTMLFormatter::dump_string_with_attrs(const char *name, boost::string_view s, const FormatterAttrs& attrs) { std::string e(name); std::string attrs_str; get_attrs_str(&attrs, attrs_str); print_spaces(); - m_ss << "
  • " << e << ": " << escape_xml_str(s.c_str()) << attrs_str << "
    • "; + m_ss << "
  • " << e << ": " << escape_xml_str(s.data()) << attrs_str << "
    • "; if (m_pretty) m_ss << "\n"; } diff --git a/ceph/src/common/HTMLFormatter.h b/ceph/src/common/HTMLFormatter.h index c59296922..5e9b200f7 100644 --- a/ceph/src/common/HTMLFormatter.h +++ b/ceph/src/common/HTMLFormatter.h @@ -18,12 +18,12 @@ namespace ceph { void dump_unsigned(const char *name, uint64_t u) override; void dump_int(const char *name, int64_t u) override; void dump_float(const char *name, double d) override; - void dump_string(const char *name, const std::string& s) override; + void dump_string(const char *name, boost::string_view s) override; std::ostream& dump_stream(const char *name) override; void dump_format_va(const char *name, const char *ns, bool quoted, const char *fmt, va_list ap) override; /* with attrs */ - void dump_string_with_attrs(const char *name, const std::string& s, const FormatterAttrs& attrs) override; + void dump_string_with_attrs(const char *name, boost::string_view s, const FormatterAttrs& attrs) override; private: template void dump_template(const char *name, T arg); diff --git a/ceph/src/common/ceph_argparse.cc b/ceph/src/common/ceph_argparse.cc index 7a2e8ea91..a89de95f6 100644 --- a/ceph/src/common/ceph_argparse.cc +++ b/ceph/src/common/ceph_argparse.cc @@ -521,22 +521,24 @@ CephInitParameters ceph_argparse_early_args static void generic_usage(bool is_server) { - cout << "\ - --conf/-c FILE read configuration from the given configuration file\n\ - --id/-i ID set ID portion of my name\n\ - --name/-n TYPE.ID set name\n\ - --cluster NAME set cluster name (default: ceph)\n\ - --setuser USER set uid to user or uid (and gid to user's gid)\n\ - --setgroup GROUP set gid to group or gid\n\ - --version show version and quit\n\ -" << std::endl; + cout << + " --conf/-c FILE read configuration from the given configuration file" << std::endl << + (is_server ? + " --id/-i ID set ID portion of my name" : + " --id ID set ID portion of my name") << std::endl << + " --name/-n TYPE.ID set name" << std::endl << + " --cluster NAME set cluster name (default: ceph)" << std::endl << + " --setuser USER set uid to user or uid (and gid to user's gid)" << std::endl << + " --setgroup GROUP set gid to group or gid" << std::endl << + " --version show version and quit" << std::endl + << std::endl; if (is_server) { - cout << "\ - -d run in foreground, log to stderr.\n\ - -f run in foreground, log to usual location.\n"; - cout << "\ - --debug_ms N set message debug level (e.g. 1)\n"; + cout << + " -d run in foreground, log to stderr" << std::endl << + " -f run in foreground, log to usual location" << std::endl << + std::endl << + " --debug_ms N set message debug level (e.g. 1)" << std::endl; } cout.flush(); diff --git a/ceph/src/common/ceph_time.h b/ceph/src/common/ceph_time.h index 8479a0d54..d3d5dfa36 100644 --- a/ceph/src/common/ceph_time.h +++ b/ceph/src/common/ceph_time.h @@ -240,6 +240,10 @@ namespace ceph { return time_point(seconds(ts.tv_sec) + nanoseconds(ts.tv_nsec)); } + static time_point zero() { + return time_point::min(); + } + // A monotonic clock's timepoints are only meaningful to the // computer on which they were generated. Thus having an // optional skew is meaningless. diff --git a/ceph/src/common/dns_resolve.cc b/ceph/src/common/dns_resolve.cc index a2b296a67..b6989da31 100644 --- a/ceph/src/common/dns_resolve.cc +++ b/ceph/src/common/dns_resolve.cc @@ -212,16 +212,17 @@ int DNSResolver::resolve_ip_addr(CephContext *cct, res_state *res, const string& u_char nsbuf[NS_PACKETSZ]; int len; - + int family = cct->_conf->ms_bind_ipv6 ? AF_INET6 : AF_INET; + int type = cct->_conf->ms_bind_ipv6 ? ns_t_aaaa : ns_t_a; #ifdef HAVE_RES_NQUERY - len = resolv_h->res_nquery(*res, hostname.c_str(), ns_c_in, ns_t_a, nsbuf, sizeof(nsbuf)); + len = resolv_h->res_nquery(*res, hostname.c_str(), ns_c_in, type, nsbuf, sizeof(nsbuf)); #else { # ifndef HAVE_THREAD_SAFE_RES_QUERY Mutex::Locker l(lock); # endif - len = resolv_h->res_query(hostname.c_str(), ns_c_in, ns_t_a, nsbuf, sizeof(nsbuf)); + len = resolv_h->res_query(hostname.c_str(), ns_c_in, type, nsbuf, sizeof(nsbuf)); } #endif if (len < 0) { @@ -250,7 +251,7 @@ int DNSResolver::resolve_ip_addr(CephContext *cct, res_state *res, const string& char addr_buf[64]; memset(addr_buf, 0, sizeof(addr_buf)); - inet_ntop(AF_INET, ns_rr_rdata(rr), addr_buf, sizeof(addr_buf)); + inet_ntop(family, ns_rr_rdata(rr), addr_buf, sizeof(addr_buf)); if (!addr->parse(addr_buf)) { lderr(cct) << "failed to parse address '" << (const char *)ns_rr_rdata(rr) << "'" << dendl; diff --git a/ceph/src/common/escape.c b/ceph/src/common/escape.c index 0d318e5f4..39c54e899 100644 --- a/ceph/src/common/escape.c +++ b/ceph/src/common/escape.c @@ -30,10 +30,10 @@ #define SGL_QUOTE_XESCAPE "'" #define DBL_QUOTE_XESCAPE """ -int escape_xml_attr_len(const char *buf) +size_t escape_xml_attr_len(const char *buf) { const char *b; - int ret = 0; + size_t ret = 0; for (b = buf; *b; ++b) { unsigned char c = *b; switch (c) { @@ -117,11 +117,10 @@ void escape_xml_attr(const char *buf, char *out) #define TAB_JESCAPE "\\t" #define NEWLINE_JESCAPE "\\n" -int escape_json_attr_len(const char *buf, int src_len) +size_t escape_json_attr_len(const char *buf, size_t src_len) { const char *b; - int ret = 0; - int i; + size_t i, ret = 0; for (i = 0, b = buf; i < src_len; ++i, ++b) { unsigned char c = *b; switch (c) { @@ -152,11 +151,11 @@ int escape_json_attr_len(const char *buf, int src_len) return ret; } -void escape_json_attr(const char *buf, int src_len, char *out) +void escape_json_attr(const char *buf, size_t src_len, char *out) { char *o = out; const char *b; - int i; + size_t i; for (i = 0, b = buf; i < src_len; ++i, ++b) { unsigned char c = *b; switch (c) { diff --git a/ceph/src/common/escape.h b/ceph/src/common/escape.h index ebdf16fd1..bea5f919b 100644 --- a/ceph/src/common/escape.h +++ b/ceph/src/common/escape.h @@ -15,6 +15,8 @@ #ifndef CEPH_RGW_ESCAPE_H #define CEPH_RGW_ESCAPE_H +#include + #ifdef __cplusplus extern "C" { #endif @@ -22,7 +24,7 @@ extern "C" { /* Returns the length of a buffer that would be needed to escape 'buf' * as an XML attrribute */ -int escape_xml_attr_len(const char *buf); +size_t escape_xml_attr_len(const char *buf); /* Escapes 'buf' as an XML attribute. Assumes that 'out' is at least long * enough to fit the output. You can find out the required length by calling @@ -33,13 +35,13 @@ void escape_xml_attr(const char *buf, char *out); /* Returns the length of a buffer that would be needed to escape 'buf' * as an JSON attrribute */ -int escape_json_attr_len(const char *buf, int src_len); +size_t escape_json_attr_len(const char *buf, size_t src_len); /* Escapes 'buf' as an JSON attribute. Assumes that 'out' is at least long * enough to fit the output. You can find out the required length by calling * escape_json_attr_len first. */ -void escape_json_attr(const char *buf, int src_len, char *out); +void escape_json_attr(const char *buf, size_t src_len, char *out); /* Note: we escape control characters. Although the XML spec doesn't actually * require this, Amazon does it in their XML responses. diff --git a/ceph/src/common/ipaddr.cc b/ceph/src/common/ipaddr.cc index 27b9cdff2..c880bfd40 100644 --- a/ceph/src/common/ipaddr.cc +++ b/ceph/src/common/ipaddr.cc @@ -90,6 +90,8 @@ const struct ifaddrs *find_ipv6_in_subnet(const struct ifaddrs *addrs, continue; struct in6_addr *cur = &((struct sockaddr_in6*)addrs->ifa_addr)->sin6_addr; + if (IN6_IS_ADDR_LINKLOCAL(cur)) + continue; netmask_ipv6(cur, prefix_len, &temp); if (IN6_ARE_ADDR_EQUAL(&temp, &want)) diff --git a/ceph/src/common/legacy_config_opts.h b/ceph/src/common/legacy_config_opts.h index de679a89e..0651b2070 100644 --- a/ceph/src/common/legacy_config_opts.h +++ b/ceph/src/common/legacy_config_opts.h @@ -801,6 +801,7 @@ OPTION(osd_max_pg_log_entries, OPT_U32) // max entries, say when degraded, befor OPTION(osd_pg_log_dups_tracked, OPT_U32) // how many versions back to track combined in both pglog's regular + dup logs OPTION(osd_force_recovery_pg_log_entries_factor, OPT_FLOAT) // max entries factor before force recovery OPTION(osd_pg_log_trim_min, OPT_U32) +OPTION(osd_pg_log_trim_max, OPT_U32) OPTION(osd_op_complaint_time, OPT_FLOAT) // how many seconds old makes an op complaint-worthy OPTION(osd_command_max_records, OPT_INT) OPTION(osd_max_pg_blocked_by, OPT_U32) // max peer osds to report that are blocking our progress @@ -818,7 +819,7 @@ OPTION(osd_debug_drop_ping_duration, OPT_INT) OPTION(osd_debug_op_order, OPT_BOOL) OPTION(osd_debug_verify_missing_on_start, OPT_BOOL) OPTION(osd_debug_scrub_chance_rewrite_digest, OPT_U64) -OPTION(osd_debug_verify_snaps_on_info, OPT_BOOL) +OPTION(osd_debug_verify_snaps, OPT_BOOL) OPTION(osd_debug_verify_stray_on_activate, OPT_BOOL) OPTION(osd_debug_skip_full_check_in_backfill_reservation, OPT_BOOL) OPTION(osd_debug_reject_backfill_probability, OPT_DOUBLE) @@ -1070,7 +1071,7 @@ OPTION(bluestore_cache_size_hdd, OPT_U64) OPTION(bluestore_cache_size_ssd, OPT_U64) OPTION(bluestore_cache_meta_ratio, OPT_DOUBLE) OPTION(bluestore_cache_kv_ratio, OPT_DOUBLE) -OPTION(bluestore_cache_kv_max, OPT_U64) // limit the maximum amount of cache for the kv store +OPTION(bluestore_cache_kv_max, OPT_INT) // limit the maximum amount of cache for the kv store OPTION(bluestore_kvbackend, OPT_STR) OPTION(bluestore_allocator, OPT_STR) // stupid | bitmap OPTION(bluestore_freelist_blocks_per_key, OPT_INT) diff --git a/ceph/src/common/options.cc b/ceph/src/common/options.cc index 0dfec7abb..3ea0af62a 100644 --- a/ceph/src/common/options.cc +++ b/ceph/src/common/options.cc @@ -2631,6 +2631,13 @@ std::vector