From 3efd99882e8c73385040d3f5c48fd014e4247be7 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fabian=20Gr=C3=BCnbichler?= Date: Tue, 28 Nov 2017 09:02:46 +0100 Subject: [PATCH] update sources to 12.2.2 --- ceph/CMakeLists.txt | 2 +- ceph/COPYING | 5 - ceph/PendingReleaseNotes | 11 + ceph/admin/doc-requirements.txt | 4 +- ceph/alpine/APKBUILD | 6 +- ceph/ceph.spec | 18 +- ceph/ceph.spec.in | 12 +- ceph/debian/ceph-osd.install | 2 + ceph/debian/ceph-osd.postinst | 1 + ceph/debian/changelog | 6 + ceph/debian/rules | 1 + ceph/doc/ceph-volume/index.rst | 44 +- ceph/doc/ceph-volume/lvm/activate.rst | 20 +- ceph/doc/ceph-volume/lvm/create.rst | 24 + ceph/doc/ceph-volume/lvm/index.rst | 6 +- ceph/doc/ceph-volume/lvm/list.rst | 173 ++++ ceph/doc/ceph-volume/lvm/prepare.rst | 148 ++- ceph/doc/ceph-volume/lvm/systemd.rst | 32 +- ceph/doc/ceph-volume/lvm/zap.rst | 19 + ceph/doc/ceph-volume/simple/activate.rst | 80 ++ ceph/doc/ceph-volume/simple/index.rst | 19 + ceph/doc/ceph-volume/simple/scan.rst | 158 +++ ceph/doc/ceph-volume/simple/systemd.rst | 28 + ceph/doc/ceph-volume/systemd.rst | 49 + ceph/doc/cephfs/mds-config-ref.rst | 14 + ceph/doc/conf.py | 3 +- ceph/doc/man/8/CMakeLists.txt | 3 +- ceph/doc/man/8/ceph-bluestore-tool.rst | 123 +++ ceph/doc/mgr/administrator.rst | 56 ++ ceph/doc/mgr/dashboard.rst | 10 +- ceph/doc/mgr/index.rst | 4 +- ceph/doc/mgr/influx.rst | 162 +++ ceph/doc/mgr/localpool.rst | 35 + ceph/doc/mgr/plugins.rst | 25 + ceph/doc/mgr/prometheus.rst | 185 +++- .../configuration/pool-pg-config-ref.rst | 9 + ceph/doc/rados/operations/health-checks.rst | 16 +- ceph/doc/scripts/gen_state_diagram.py | 28 +- ceph/etc/default/ceph | 8 - ceph/etc/sysconfig/ceph | 8 - ceph/qa/cephfs/clusters/3-mds.yaml | 4 +- ceph/qa/cephfs/clusters/9-mds.yaml | 4 +- .../bluestore-comp-ec-root.yaml | 28 + .../cephfs/objectstore-ec/bluestore-comp.yaml | 23 + .../objectstore-ec/bluestore-ec-root.yaml | 42 + ceph/qa/cephfs/objectstore-ec/bluestore.yaml | 38 + .../cephfs/objectstore-ec/filestore-xfs.yaml | 15 + ceph/qa/distros/all/centos_7.4.yaml | 2 + ceph/qa/distros/supported/centos_latest.yaml | 2 +- ceph/qa/releases/luminous-with-mgr.yaml | 3 +- ceph/qa/releases/luminous.yaml | 1 + ceph/qa/standalone/mon/osd-pool-create.sh | 2 +- .../special}/ceph_objectstore_tool.py | 49 +- .../smoke/basic/1-distros/centos_7.3.yaml | 2 - .../smoke/basic/1-distros/centos_latest.yaml | 1 + .../smoke/basic/1-distros/ubuntu_16.04.yaml | 2 - .../smoke/basic/1-distros/ubuntu_latest.yaml | 1 + .../smoke/basic/2-ceph/ceph_ansible.yaml | 32 + .../smoke/basic/2-config/ceph_ansible.yaml | 22 - .../3-config/bluestore_with_dmcrypt.yaml | 8 + .../smoke/basic/3-config/dmcrypt_off.yaml | 7 + .../smoke/basic/3-config/dmcrypt_on.yaml | 7 + .../ceph-ansible/smoke/basic/3-tasks/cls.yaml | 7 - .../ceph-admin-commands.yaml | 0 .../rbd_import_export.yaml | 0 .../smoke/basic/4-tasks/rest.yaml | 15 + ceph/qa/suites/fs/32bits/objectstore | 1 - ceph/qa/suites/fs/32bits/objectstore-ec | 1 + .../clusters/4-remote-clients.yaml | 4 +- .../objectstore/bluestore-ec-root.yaml | 1 + ceph/qa/suites/fs/basic_workload/objectstore | 1 - .../suites/fs/basic_workload/objectstore-ec | 1 + .../multiclient/clusters/three_clients.yaml | 2 +- .../fs/multiclient/clusters/two_clients.yaml | 2 +- ceph/qa/suites/fs/multiclient/objectstore | 1 - ceph/qa/suites/fs/multiclient/objectstore-ec | 1 + .../fs/multifs/clusters/2-remote-clients.yaml | 4 +- ceph/qa/suites/fs/multifs/objectstore | 1 - ceph/qa/suites/fs/multifs/objectstore-ec | 1 + ceph/qa/suites/fs/permission/objectstore | 1 - ceph/qa/suites/fs/permission/objectstore-ec | 1 + ceph/qa/suites/fs/snaps/objectstore | 1 - ceph/qa/suites/fs/snaps/objectstore-ec | 1 + ceph/qa/suites/fs/thrash/objectstore | 1 - ceph/qa/suites/fs/thrash/objectstore-ec | 1 + ceph/qa/suites/fs/traceless/objectstore | 1 - ceph/qa/suites/fs/traceless/objectstore-ec | 1 + ceph/qa/suites/fs/verify/objectstore | 1 - ceph/qa/suites/fs/verify/objectstore-ec | 1 + ceph/qa/suites/kcephfs/cephfs/objectstore | 1 - ceph/qa/suites/kcephfs/cephfs/objectstore-ec | 1 + .../suites/kcephfs/mixed-clients/objectstore | 1 - .../kcephfs/mixed-clients/objectstore-ec | 1 + .../recovery/clusters/4-remote-clients.yaml | 4 +- ceph/qa/suites/kcephfs/recovery/objectstore | 1 - .../qa/suites/kcephfs/recovery/objectstore-ec | 1 + ceph/qa/suites/kcephfs/thrash/objectstore | 1 - ceph/qa/suites/kcephfs/thrash/objectstore-ec | 1 + ceph/qa/suites/multimds/basic/objectstore | 1 - ceph/qa/suites/multimds/basic/objectstore-ec | 1 + ceph/qa/suites/multimds/thrash/objectstore | 1 - ceph/qa/suites/multimds/thrash/objectstore-ec | 1 + ceph/qa/suites/multimds/verify/objectstore | 1 - ceph/qa/suites/multimds/verify/objectstore-ec | 1 + ceph/qa/suites/rados/basic/d-require-luminous | 1 - .../basic/d-require-luminous/at-end.yaml | 33 + .../basic/d-require-luminous/at-mkfs.yaml} | 0 .../suites/rados/mgr/clusters/2-node-mgr.yaml | 2 +- ceph/qa/suites/rados/mgr/tasks/dashboard.yaml | 16 + .../rados/mgr/tasks/module_selftest.yaml | 19 + ceph/qa/suites/rados/mgr/tasks/workunits.yaml | 16 + .../suites/rados/monthrash/d-require-luminous | 2 +- ceph/qa/suites/rados/rest/mgr-restful.yaml | 3 + .../basic/tasks => rados/rest}/rest_test.yaml | 13 +- .../all/admin_socket_output.yaml | 1 + .../all/max-pg-per-osd.from-mon.yaml | 26 + .../all/max-pg-per-osd.from-primary.yaml | 31 + .../all/max-pg-per-osd.from-replica.yaml | 31 + .../rados/singleton/all/mon-seesaw.yaml | 4 + .../singleton/all/recovery-preemption.yaml | 51 + .../at-mkfs-balancer-crush-compat.yaml | 11 + .../at-mkfs-balancer-upmap.yaml | 11 + .../qa/suites/rados/verify/d-require-luminous | 2 +- .../suites/rbd/basic/tasks/rbd_cls_tests.yaml | 2 + ceph/qa/suites/rgw/hadoop-s3a/s3a-hadoop.yaml | 17 +- .../suites/rgw/multifs/tasks/rgw_s3tests.yaml | 2 +- .../rgw/thrash/workload/rgw_s3tests.yaml | 2 +- .../suites/rgw/verify/tasks/rgw_s3tests.yaml | 2 +- .../+ => jewel-x/ceph-deploy/%} | 0 .../ceph-deploy/distros/centos_latest.yaml | 1 + .../ceph-deploy/distros/ubuntu_latest.yaml | 1 + .../jewel-x/ceph-deploy/jewel-luminous.yaml | 82 ++ .../jewel-x/parallel/0-cluster/start.yaml | 1 + .../parallel/1-jewel-install/jewel.yaml | 19 + .../parallel/2-workload/blogbench.yaml | 2 +- .../upgrade/jewel-x/parallel/4-luminous.yaml | 24 +- .../upgrade/jewel-x/parallel/5-workload.yaml | 11 + .../parallel/6-luminous-with-mgr.yaml} | 0 .../jewel-x/parallel/6.5-crush-compat.yaml | 8 + .../jewel-x/parallel/7-final-workload/+ | 0 .../parallel/7-final-workload}/blogbench.yaml | 4 +- .../rados-snaps-few-objects.yaml | 2 +- .../7-final-workload}/rados_loadgenmix.yaml | 2 +- .../rados_mon_thrash.yaml | 2 +- .../parallel/7-final-workload}/rbd_cls.yaml | 2 +- .../7-final-workload}/rbd_import_export.yaml | 2 +- .../parallel/7-final-workload}/rgw_swift.yaml | 4 +- .../jewel-x/parallel/8-jewel-workload.yaml | 1 + .../point-to-point-upgrade.yaml | 13 +- .../stress-split/6.5-crush-compat.yaml | 1 + .../kraken-x/ceph-deploy/kraken-luminous.yaml | 61 ++ .../kraken-x/parallel/0-cluster/start.yaml | 1 + .../upgrade/kraken-x/parallel/4-luminous.yaml | 4 + .../upgrade/kraken-x/parallel/5-workload.yaml | 11 + .../parallel/6-luminous-with-mgr.yaml | 1 + .../kraken-x/parallel/7-final-workload/+ | 0 .../parallel/7-final-workload}/blogbench.yaml | 0 .../rados-snaps-few-objects.yaml | 0 .../7-final-workload}/rados_loadgenmix.yaml | 0 .../rados_mon_thrash.yaml | 0 .../parallel/7-final-workload}/rbd_cls.yaml | 0 .../7-final-workload}/rbd_import_export.yaml | 0 .../parallel/7-final-workload}/rgw_swift.yaml | 0 .../luminous-x/parallel/0-cluster/start.yaml | 1 + .../rbd_import_export_no_upgrated.yaml | 13 + ...t.yaml => rbd_import_export_upgrated.yaml} | 3 +- .../stress-split-erasure-code/1-ceph-install | 1 + ceph/qa/tasks/ceph.py | 30 +- ceph/qa/tasks/ceph_deploy.py | 158 ++- ceph/qa/tasks/ceph_manager.py | 47 +- ceph/qa/tasks/ceph_objectstore_tool.py | 2 +- ceph/qa/tasks/cephfs/filesystem.py | 22 +- ceph/qa/tasks/cephfs/test_client_limits.py | 38 +- ceph/qa/tasks/cephfs/test_volume_client.py | 119 ++- ceph/qa/tasks/divergent_priors2.py | 8 +- ceph/qa/tasks/mgr/mgr_test_case.py | 93 +- ceph/qa/tasks/mgr/test_dashboard.py | 70 ++ ceph/qa/tasks/mgr/test_module_selftest.py | 74 ++ ceph/qa/tasks/osd_max_pg_per_osd.py | 126 +++ ceph/qa/tasks/reg11184.py | 11 +- ceph/qa/tasks/s3a_hadoop.py | 4 +- ceph/qa/tasks/thrashosds.py | 2 +- ceph/qa/tasks/util/rados.py | 4 +- ceph/qa/workunits/ceph-disk/ceph-disk-test.py | 2 +- ceph/qa/workunits/ceph-disk/ceph-disk.sh | 2 +- ceph/qa/workunits/cephtool/test.sh | 18 +- ceph/qa/workunits/cls/test_cls_journal.sh | 6 + ceph/qa/workunits/mgr/test_localpool.sh | 21 + ceph/qa/workunits/rados/test_rados_tool.sh | 2 +- ceph/qa/workunits/rbd/rbd_mirror.sh | 12 + ceph/qa/workunits/rbd/rbd_mirror_helpers.sh | 10 + ceph/selinux/ceph.te | 2 +- ceph/src/.git_version | 4 +- ceph/src/90-ceph-osd.conf | 1 + ceph/src/CMakeLists.txt | 18 +- ceph/src/arch/arm.c | 5 +- ceph/src/ceph-disk/ceph_disk/main.py | 162 +-- ceph/src/ceph-disk/tox.ini | 2 +- .../ceph-volume/ceph_volume/api/__init__.py | 3 + .../{devices/lvm/api.py => api/lvm.py} | 108 +- .../src/ceph-volume/ceph_volume/decorators.py | 3 + .../ceph_volume/devices/__init__.py | 2 +- .../ceph_volume/devices/lvm/activate.py | 114 ++- .../ceph_volume/devices/lvm/common.py | 24 +- .../ceph_volume/devices/lvm/create.py | 4 + .../ceph_volume/devices/lvm/listing.py | 244 +++++ .../ceph_volume/devices/lvm/main.py | 4 + .../ceph_volume/devices/lvm/prepare.py | 232 +++-- .../ceph_volume/devices/lvm/trigger.py | 2 +- .../ceph_volume/devices/lvm/zap.py | 107 ++ .../ceph_volume/devices/simple/__init__.py | 1 + .../ceph_volume/devices/simple/activate.py | 152 +++ .../ceph_volume/devices/simple/main.py | 41 + .../ceph_volume/devices/simple/scan.py | 206 ++++ .../ceph_volume/devices/simple/trigger.py | 70 ++ ceph/src/ceph-volume/ceph_volume/main.py | 2 +- ceph/src/ceph-volume/ceph_volume/process.py | 47 +- .../ceph_volume/systemd/systemctl.py | 15 + .../lvm/test_api.py => api/test_lvm.py} | 66 +- .../ceph-volume/ceph_volume/tests/conftest.py | 34 +- .../tests/devices/lvm/test_activate.py | 48 +- .../tests/devices/lvm/test_listing.py | 176 ++++ .../tests/devices/lvm/test_prepare.py | 9 +- .../tests/devices/simple/test_activate.py | 23 + .../tests/devices/simple/test_scan.py | 52 + .../tests/devices/simple/test_trigger.py | 45 + .../ceph_volume/tests/devices/test_zap.py | 17 + .../functional/centos7/create/Vagrantfile | 1 - .../lvm/centos7/bluestore/create/Vagrantfile | 1 + .../centos7/bluestore/create/group_vars/all | 28 + .../centos7/bluestore}/create/hosts | 0 .../lvm/centos7/bluestore/create/setup.yml | 1 + .../bluestore}/create/vagrant_variables.yml | 0 .../lvm/centos7/filestore/create/Vagrantfile | 1 + .../centos7/filestore}/create/group_vars/all | 2 + .../centos7/filestore}/create/hosts | 0 .../lvm/centos7/filestore/create/setup.yml | 1 + .../filestore/create/vagrant_variables.yml | 56 ++ .../lvm/playbooks/setup_partitions.yml | 27 + .../ceph_volume/tests/functional/lvm/tox.ini | 59 ++ .../lvm/xenial/bluestore/create/Vagrantfile | 1 + .../xenial/bluestore/create/group_vars/all | 28 + .../lvm/xenial/bluestore/create/hosts | 5 + .../lvm/xenial/bluestore/create/setup.yml | 1 + .../bluestore/create/vagrant_variables.yml | 56 ++ .../lvm/xenial/filestore/create/Vagrantfile | 1 + .../xenial/filestore}/create/group_vars/all | 2 + .../lvm/xenial/filestore/create/hosts | 5 + .../lvm/xenial/filestore/create/setup.yml | 1 + .../filestore}/create/vagrant_variables.yml | 0 .../centos7/bluestore/activate/Vagrantfile | 1 + .../centos7/bluestore/activate/group_vars/all | 19 + .../bluestore/activate/host_vars/osd0.yml | 7 + .../bluestore/activate/host_vars/osd1.yml | 6 + .../simple/centos7/bluestore/activate/hosts | 9 + .../centos7/bluestore/activate/test.yml | 31 + .../bluestore/activate/vagrant_variables.yml | 73 ++ .../centos7/filestore/activate/Vagrantfile | 1 + .../centos7/filestore/activate/group_vars/all | 19 + .../filestore/activate/host_vars/osd0.yml | 7 + .../filestore/activate/host_vars/osd1.yml | 6 + .../simple/centos7/filestore/activate/hosts | 9 + .../centos7/filestore/activate/test.yml | 31 + .../filestore/activate/vagrant_variables.yml | 73 ++ .../tests/functional/{ => simple}/tox.ini | 24 +- .../xenial/bluestore/activate/Vagrantfile | 1 + .../xenial/bluestore/activate/group_vars/all | 19 + .../bluestore/activate/host_vars/osd0.yml | 7 + .../bluestore/activate/host_vars/osd1.yml | 6 + .../simple/xenial/bluestore/activate/hosts | 9 + .../simple/xenial/bluestore/activate/test.yml | 31 + .../bluestore/activate/vagrant_variables.yml | 73 ++ .../xenial/filestore/activate/Vagrantfile | 1 + .../xenial/filestore/activate/group_vars/all | 19 + .../filestore/activate/host_vars/osd0.yml | 7 + .../filestore/activate/host_vars/osd1.yml | 6 + .../simple/xenial/filestore/activate/hosts | 9 + .../simple/xenial/filestore/activate/test.yml | 31 + .../filestore/activate/vagrant_variables.yml | 73 ++ .../functional/xenial/create/Vagrantfile | 1 - .../tests/util/test_arg_validators.py | 31 +- .../ceph_volume/tests/util/test_system.py | 141 ++- .../ceph_volume/util/arg_validators.py | 44 + ceph/src/ceph-volume/ceph_volume/util/disk.py | 159 +++ .../ceph-volume/ceph_volume/util/prepare.py | 108 +- .../ceph-volume/ceph_volume/util/system.py | 140 ++- ceph/src/ceph.in | 4 +- ceph/src/ceph_mgr.cc | 5 + ceph/src/ceph_mon.cc | 20 +- ceph/src/ceph_osd.cc | 46 +- ceph/src/client/Client.cc | 23 +- ceph/src/client/Client.h | 1 - ceph/src/cls/journal/cls_journal.cc | 2 + ceph/src/cls/rbd/cls_rbd.cc | 91 +- ceph/src/cls/rgw/cls_rgw.cc | 46 +- ceph/src/cls/user/cls_user.cc | 1 - ceph/src/cls/user/cls_user_types.h | 11 +- ceph/src/common/AsyncReserver.h | 162 ++- ceph/src/common/LogClient.cc | 2 +- ceph/src/common/Timer.cc | 8 +- ceph/src/common/Timer.h | 4 +- ceph/src/common/bit_vector.hpp | 189 +++- ceph/src/common/buffer.cc | 79 +- ceph/src/common/ceph_context.cc | 2 +- ceph/src/common/common_init.cc | 4 + ceph/src/common/config.cc | 15 +- ceph/src/common/legacy_config_opts.h | 28 - ceph/src/common/options.cc | 199 +++- ceph/src/common/perf_counters.cc | 11 +- ceph/src/common/perf_counters.h | 155 +-- ceph/src/common/pick_address.cc | 102 +- ceph/src/common/pick_address.h | 7 + ceph/src/common/subsys.h | 1 + ceph/src/crush/CrushTreeDumper.h | 6 +- ceph/src/crush/CrushWrapper.cc | 192 ++-- ceph/src/crush/CrushWrapper.h | 78 +- ceph/src/include/buffer.h | 10 +- ceph/src/include/rados/rgw_file.h | 12 +- ceph/src/include/sock_compat.h | 13 + ceph/src/journal/JournalMetadata.cc | 6 +- ceph/src/journal/ObjectPlayer.cc | 13 +- ceph/src/journal/ObjectPlayer.h | 6 - ceph/src/journal/ObjectRecorder.cc | 10 +- ceph/src/journal/ObjectRecorder.h | 10 +- ceph/src/kv/KeyValueDB.h | 10 + ceph/src/kv/LevelDBStore.h | 5 + ceph/src/kv/RocksDBStore.h | 5 + ceph/src/librbd/ObjectMap.cc | 17 +- ceph/src/librbd/ObjectMap.h | 13 +- ceph/src/librbd/api/Mirror.cc | 8 +- ceph/src/librbd/io/ObjectRequest.cc | 10 + ceph/src/librbd/io/ObjectRequest.h | 5 + ceph/src/librbd/object_map/UpdateRequest.cc | 83 +- ceph/src/librbd/object_map/UpdateRequest.h | 26 +- .../librbd/operation/SnapshotCreateRequest.cc | 4 +- .../librbd/operation/SnapshotRemoveRequest.cc | 2 +- ceph/src/librbd/operation/TrimRequest.cc | 227 ++--- ceph/src/librbd/operation/TrimRequest.h | 76 +- ceph/src/mds/Beacon.cc | 22 +- ceph/src/mds/Beacon.h | 3 +- ceph/src/mds/CInode.h | 2 +- ceph/src/mds/FSMap.cc | 16 +- ceph/src/mds/FSMap.h | 1 + ceph/src/mds/MDCache.cc | 4 +- ceph/src/mds/MDSDaemon.cc | 21 +- ceph/src/mds/MDSDaemon.h | 3 +- ceph/src/mds/MDSMap.cc | 23 +- ceph/src/mds/MDSMap.h | 2 +- ceph/src/mds/MDSRank.cc | 51 +- ceph/src/mds/PurgeQueue.cc | 41 +- ceph/src/mds/PurgeQueue.h | 6 +- ceph/src/mds/Server.cc | 29 +- ceph/src/messages/MMgrBeacon.h | 18 +- ceph/src/messages/MMgrConfigure.h | 12 +- ceph/src/messages/MMgrReport.h | 13 +- ceph/src/messages/MOSDMap.h | 11 +- ceph/src/messages/MOSDPGTemp.h | 18 +- ceph/src/mgr/ActivePyModule.cc | 225 +++++ .../mgr/{MgrPyModule.h => ActivePyModule.h} | 50 +- .../mgr/{PyModules.cc => ActivePyModules.cc} | 472 ++++----- .../mgr/{PyModules.h => ActivePyModules.h} | 77 +- ceph/src/mgr/BaseMgrModule.cc | 636 ++++++++++++ ceph/src/mgr/BaseMgrModule.h | 7 + ceph/src/mgr/BaseMgrStandbyModule.cc | 161 +++ ceph/src/mgr/BaseMgrStandbyModule.h | 7 + ceph/src/mgr/DaemonServer.cc | 195 +++- ceph/src/mgr/DaemonServer.h | 21 +- ceph/src/mgr/DaemonState.cc | 33 +- ceph/src/mgr/DaemonState.h | 51 +- ceph/src/mgr/Gil.cc | 79 ++ ceph/src/mgr/Gil.h | 90 +- ceph/src/mgr/Mgr.cc | 51 +- ceph/src/mgr/Mgr.h | 13 +- ceph/src/mgr/MgrClient.cc | 68 +- ceph/src/mgr/MgrClient.h | 1 + ceph/src/mgr/MgrCommands.h | 5 + ceph/src/mgr/MgrPyModule.cc | 371 ------- ceph/src/mgr/MgrSession.h | 2 + ceph/src/mgr/MgrStandby.cc | 73 +- ceph/src/mgr/MgrStandby.h | 2 + ceph/src/mgr/PyModuleRegistry.cc | 450 +++++++++ ceph/src/mgr/PyModuleRegistry.h | 173 ++++ ceph/src/mgr/PyModuleRunner.cc | 97 ++ ceph/src/mgr/PyModuleRunner.h | 76 ++ ceph/src/mgr/PyOSDMap.cc | 589 +++++++++++ ceph/src/mgr/PyOSDMap.h | 20 + ceph/src/mgr/PyState.cc | 490 --------- ceph/src/mgr/PyState.h | 12 - ceph/src/mgr/StandbyPyModules.cc | 200 ++++ ceph/src/mgr/StandbyPyModules.h | 149 +++ ceph/src/mon/AuthMonitor.cc | 12 +- ceph/src/mon/Elector.cc | 10 +- ceph/src/mon/LogMonitor.cc | 17 +- ceph/src/mon/MDSMonitor.cc | 18 +- ceph/src/mon/MDSMonitor.h | 3 +- ceph/src/mon/MgrMap.h | 16 +- ceph/src/mon/MgrMonitor.cc | 103 +- ceph/src/mon/MgrMonitor.h | 6 + ceph/src/mon/MonCommands.h | 12 +- ceph/src/mon/MonMap.cc | 21 +- ceph/src/mon/Monitor.cc | 114 ++- ceph/src/mon/MonitorDBStore.h | 12 + ceph/src/mon/OSDMonitor.cc | 461 +++++++-- ceph/src/mon/OSDMonitor.h | 6 +- ceph/src/mon/PGMap.cc | 60 +- ceph/src/mon/Paxos.cc | 72 +- ceph/src/mon/PaxosService.cc | 6 +- ceph/src/msg/Messenger.h | 45 +- ceph/src/msg/async/AsyncConnection.h | 1 - ceph/src/msg/async/PosixStack.cc | 79 +- ceph/src/msg/async/net_handler.cc | 2 +- ceph/src/msg/simple/Pipe.cc | 87 +- ceph/src/msg/simple/Pipe.h | 9 - ceph/src/os/ObjectMap.h | 2 +- ceph/src/os/ObjectStore.h | 3 + .../src/os/bluestore/BitmapFreelistManager.cc | 51 +- ceph/src/os/bluestore/BitmapFreelistManager.h | 5 +- ceph/src/os/bluestore/BlueFS.cc | 22 + ceph/src/os/bluestore/BlueFS.h | 2 + ceph/src/os/bluestore/BlueStore.cc | 604 +++++++----- ceph/src/os/bluestore/BlueStore.h | 34 +- ceph/src/os/bluestore/FreelistManager.h | 5 +- ceph/src/os/bluestore/KernelDevice.cc | 5 + ceph/src/os/bluestore/bluestore_tool.cc | 342 +++++-- ceph/src/os/bluestore/bluestore_types.cc | 20 +- ceph/src/os/bluestore/bluestore_types.h | 4 + ceph/src/os/filestore/DBObjectMap.cc | 139 +-- ceph/src/os/filestore/DBObjectMap.h | 24 +- ceph/src/os/filestore/FileJournal.cc | 3 +- ceph/src/os/filestore/FileStore.cc | 3 +- ceph/src/osd/OSD.cc | 184 +++- ceph/src/osd/OSD.h | 11 +- ceph/src/osd/OSDMap.cc | 145 ++- ceph/src/osd/OSDMap.h | 21 +- ceph/src/osd/PG.cc | 114 ++- ceph/src/osd/PG.h | 99 +- ceph/src/osd/PrimaryLogPG.cc | 36 +- ceph/src/osd/PrimaryLogPG.h | 6 +- ceph/src/osd/ReplicatedBackend.h | 1 - ceph/src/osd/Watch.cc | 12 +- ceph/src/osd/osd_types.cc | 15 +- ceph/src/osd/osd_types.h | 16 +- ceph/src/osdc/ObjectCacher.cc | 23 +- ceph/src/osdc/ObjectCacher.h | 5 +- ceph/src/pybind/ceph_volume_client.py | 72 +- ceph/src/pybind/mgr/balancer/__init__.py | 2 + ceph/src/pybind/mgr/balancer/module.py | 933 ++++++++++++++++++ ceph/src/pybind/mgr/dashboard/base.html | 127 +-- ceph/src/pybind/mgr/dashboard/clients.html | 2 +- ceph/src/pybind/mgr/dashboard/filesystem.html | 9 +- ceph/src/pybind/mgr/dashboard/health.html | 23 +- ceph/src/pybind/mgr/dashboard/module.py | 117 ++- ceph/src/pybind/mgr/dashboard/osd_perf.html | 2 +- ceph/src/pybind/mgr/dashboard/osds.html | 2 +- ceph/src/pybind/mgr/dashboard/rbd_iscsi.html | 2 +- ceph/src/pybind/mgr/dashboard/rbd_ls.py | 4 +- .../pybind/mgr/dashboard/rbd_mirroring.html | 2 +- ceph/src/pybind/mgr/dashboard/rbd_pool.html | 2 +- ceph/src/pybind/mgr/dashboard/servers.html | 2 +- ceph/src/pybind/mgr/dashboard/standby.html | 15 + ceph/src/pybind/mgr/influx/__init__.py | 1 + ceph/src/pybind/mgr/influx/module.py | 162 +++ ceph/src/pybind/mgr/localpool/__init__.py | 2 + ceph/src/pybind/mgr/localpool/module.py | 92 ++ ceph/src/pybind/mgr/mgr_module.py | 338 ++++++- ceph/src/pybind/mgr/prometheus/module.py | 354 +++++-- ceph/src/pybind/mgr/restful/module.py | 22 +- ceph/src/pybind/mgr/selftest/__init__.py | 3 + ceph/src/pybind/mgr/selftest/module.py | 217 ++++ ceph/src/pybind/mgr/status/module.py | 28 +- ceph/src/pybind/mgr/zabbix/module.py | 10 +- ceph/src/rbdmap | 16 +- ceph/src/rgw/rgw_admin.cc | 8 + ceph/src/rgw/rgw_auth_s3.h | 14 +- ceph/src/rgw/rgw_basic_types.h | 2 - ceph/src/rgw/rgw_bucket.cc | 29 +- ceph/src/rgw/rgw_bucket.h | 14 +- ceph/src/rgw/rgw_common.cc | 16 +- ceph/src/rgw/rgw_common.h | 56 +- ceph/src/rgw/rgw_crypt.cc | 187 +++- ceph/src/rgw/rgw_data_sync.cc | 14 +- ceph/src/rgw/rgw_file.cc | 202 +++- ceph/src/rgw/rgw_file.h | 163 ++- ceph/src/rgw/rgw_iam_policy.cc | 12 +- ceph/src/rgw/rgw_iam_policy.h | 3 - ceph/src/rgw/rgw_json_enc.cc | 1 + ceph/src/rgw/rgw_keystone.h | 7 +- ceph/src/rgw/rgw_lc.cc | 12 +- ceph/src/rgw/rgw_op.cc | 290 ++++-- ceph/src/rgw/rgw_op.h | 144 ++- ceph/src/rgw/rgw_quota.cc | 5 +- ceph/src/rgw/rgw_rados.cc | 44 +- ceph/src/rgw/rgw_rados.h | 10 +- ceph/src/rgw/rgw_reshard.cc | 12 +- ceph/src/rgw/rgw_rest.h | 19 + ceph/src/rgw/rgw_rest_swift.cc | 255 +++-- ceph/src/rgw/rgw_rest_swift.h | 13 +- ceph/src/rgw/rgw_rest_user.cc | 4 + ceph/src/rgw/rgw_swift_auth.cc | 10 +- ceph/src/rgw/rgw_swift_auth.h | 15 +- ceph/src/rgw/rgw_torrent.cc | 15 +- ceph/src/rgw/rgw_torrent.h | 6 +- ceph/src/rgw/rgw_user.cc | 23 +- ceph/src/rgw/rgw_user.h | 5 + ceph/src/test/CMakeLists.txt | 15 + ceph/src/test/cli/crushtool/build.t | 2 +- ceph/src/test/cli/osdmaptool/help.t | 1 + .../test/cli/osdmaptool/missing-argument.t | 1 + ceph/src/test/cli/osdmaptool/upmap-out.t | 23 + ceph/src/test/cls_journal/CMakeLists.txt | 18 + ceph/src/test/cls_journal/test_cls_journal.cc | 20 +- ceph/src/test/common/test_bit_vector.cc | 28 + ceph/src/test/daemon_config.cc | 117 +-- ceph/src/test/librados/misc.cc | 22 + ceph/src/test/librbd/CMakeLists.txt | 1 + ceph/src/test/librbd/mock/MockImageCtx.h | 5 + ceph/src/test/librbd/mock/MockObjectMap.h | 18 +- .../object_map/test_mock_UpdateRequest.cc | 63 +- .../test_mock_SnapshotCreateRequest.cc | 4 +- .../test_mock_SnapshotRemoveRequest.cc | 2 +- .../librbd/operation/test_mock_TrimRequest.cc | 496 ++++++++++ ceph/src/test/librgw_file.cc | 4 +- ceph/src/test/librgw_file_aw.cc | 4 +- ceph/src/test/librgw_file_cd.cc | 4 +- ceph/src/test/librgw_file_gp.cc | 4 +- ceph/src/test/librgw_file_marker.cc | 488 +++++++++ ceph/src/test/librgw_file_nfsns.cc | 4 +- ceph/src/test/mon/PGMap.cc | 2 +- ceph/src/test/perf_counters.cc | 2 +- ceph/src/test/perf_local.cc | 5 +- .../test_mock_BootstrapRequest.cc | 2 + ceph/src/test/rbd_mirror/mock/MockSafeTimer.h | 2 +- .../rbd_mirror/test_mock_ImageReplayer.cc | 8 +- .../rbd_mirror/test_mock_InstanceReplayer.cc | 9 +- .../test/rbd_mirror/test_mock_PoolWatcher.cc | 17 +- ceph/src/test/rgw/rgw_multi/tests.py | 22 + ceph/src/test/test_ipaddr.cc | 51 + ceph/src/tools/CMakeLists.txt | 4 +- ceph/src/tools/ceph_kvstore_tool.cc | 23 +- ceph/src/tools/ceph_monstore_tool.cc | 35 + ceph/src/tools/ceph_objectstore_tool.cc | 82 +- ceph/src/tools/ceph_osdomap_tool.cc | 25 +- ceph/src/tools/crushtool.cc | 17 +- ceph/src/tools/monmaptool.cc | 6 +- ceph/src/tools/osdmaptool.cc | 13 + ceph/src/tools/rbd/action/MirrorImage.cc | 39 + ceph/src/tools/rbd/action/MirrorPool.cc | 67 +- ceph/src/tools/rbd_mirror/ImageReplayer.cc | 252 +++-- ceph/src/tools/rbd_mirror/ImageReplayer.h | 2 + ceph/src/tools/rbd_mirror/PoolReplayer.cc | 36 +- ceph/src/tools/rbd_mirror/PoolReplayer.h | 3 +- ceph/src/tools/rbd_mirror/PoolWatcher.cc | 9 +- .../image_replayer/BootstrapRequest.cc | 15 +- .../rbd_mirror/image_sync/ImageCopyRequest.cc | 5 +- ceph/src/vstart.sh | 12 +- ceph/systemd/ceph-rbd-mirror@.service | 1 + 556 files changed, 18785 insertions(+), 4865 deletions(-) create mode 100644 ceph/doc/ceph-volume/lvm/create.rst create mode 100644 ceph/doc/ceph-volume/lvm/list.rst create mode 100644 ceph/doc/ceph-volume/lvm/zap.rst create mode 100644 ceph/doc/ceph-volume/simple/activate.rst create mode 100644 ceph/doc/ceph-volume/simple/index.rst create mode 100644 ceph/doc/ceph-volume/simple/scan.rst create mode 100644 ceph/doc/ceph-volume/simple/systemd.rst create mode 100644 ceph/doc/ceph-volume/systemd.rst create mode 100644 ceph/doc/man/8/ceph-bluestore-tool.rst create mode 100644 ceph/doc/mgr/influx.rst create mode 100644 ceph/doc/mgr/localpool.rst create mode 100644 ceph/qa/cephfs/objectstore-ec/bluestore-comp-ec-root.yaml create mode 100644 ceph/qa/cephfs/objectstore-ec/bluestore-comp.yaml create mode 100644 ceph/qa/cephfs/objectstore-ec/bluestore-ec-root.yaml create mode 100644 ceph/qa/cephfs/objectstore-ec/bluestore.yaml create mode 100644 ceph/qa/cephfs/objectstore-ec/filestore-xfs.yaml create mode 100644 ceph/qa/distros/all/centos_7.4.yaml rename ceph/{src/test => qa/standalone/special}/ceph_objectstore_tool.py (97%) delete mode 100644 ceph/qa/suites/ceph-ansible/smoke/basic/1-distros/centos_7.3.yaml create mode 120000 ceph/qa/suites/ceph-ansible/smoke/basic/1-distros/centos_latest.yaml delete mode 100644 ceph/qa/suites/ceph-ansible/smoke/basic/1-distros/ubuntu_16.04.yaml create mode 120000 ceph/qa/suites/ceph-ansible/smoke/basic/1-distros/ubuntu_latest.yaml create mode 100644 ceph/qa/suites/ceph-ansible/smoke/basic/2-ceph/ceph_ansible.yaml delete mode 100644 ceph/qa/suites/ceph-ansible/smoke/basic/2-config/ceph_ansible.yaml create mode 100644 ceph/qa/suites/ceph-ansible/smoke/basic/3-config/bluestore_with_dmcrypt.yaml create mode 100644 ceph/qa/suites/ceph-ansible/smoke/basic/3-config/dmcrypt_off.yaml create mode 100644 ceph/qa/suites/ceph-ansible/smoke/basic/3-config/dmcrypt_on.yaml delete mode 100644 ceph/qa/suites/ceph-ansible/smoke/basic/3-tasks/cls.yaml rename ceph/qa/suites/ceph-ansible/smoke/basic/{3-tasks => 4-tasks}/ceph-admin-commands.yaml (100%) rename ceph/qa/suites/ceph-ansible/smoke/basic/{3-tasks => 4-tasks}/rbd_import_export.yaml (100%) create mode 100644 ceph/qa/suites/ceph-ansible/smoke/basic/4-tasks/rest.yaml delete mode 120000 ceph/qa/suites/fs/32bits/objectstore create mode 120000 ceph/qa/suites/fs/32bits/objectstore-ec create mode 120000 ceph/qa/suites/fs/basic_functional/objectstore/bluestore-ec-root.yaml delete mode 120000 ceph/qa/suites/fs/basic_workload/objectstore create mode 120000 ceph/qa/suites/fs/basic_workload/objectstore-ec delete mode 120000 ceph/qa/suites/fs/multiclient/objectstore create mode 120000 ceph/qa/suites/fs/multiclient/objectstore-ec delete mode 120000 ceph/qa/suites/fs/multifs/objectstore create mode 120000 ceph/qa/suites/fs/multifs/objectstore-ec delete mode 120000 ceph/qa/suites/fs/permission/objectstore create mode 120000 ceph/qa/suites/fs/permission/objectstore-ec delete mode 120000 ceph/qa/suites/fs/snaps/objectstore create mode 120000 ceph/qa/suites/fs/snaps/objectstore-ec delete mode 120000 ceph/qa/suites/fs/thrash/objectstore create mode 120000 ceph/qa/suites/fs/thrash/objectstore-ec delete mode 120000 ceph/qa/suites/fs/traceless/objectstore create mode 120000 ceph/qa/suites/fs/traceless/objectstore-ec delete mode 120000 ceph/qa/suites/fs/verify/objectstore create mode 120000 ceph/qa/suites/fs/verify/objectstore-ec delete mode 120000 ceph/qa/suites/kcephfs/cephfs/objectstore create mode 120000 ceph/qa/suites/kcephfs/cephfs/objectstore-ec delete mode 120000 ceph/qa/suites/kcephfs/mixed-clients/objectstore create mode 120000 ceph/qa/suites/kcephfs/mixed-clients/objectstore-ec delete mode 120000 ceph/qa/suites/kcephfs/recovery/objectstore create mode 120000 ceph/qa/suites/kcephfs/recovery/objectstore-ec delete mode 120000 ceph/qa/suites/kcephfs/thrash/objectstore create mode 120000 ceph/qa/suites/kcephfs/thrash/objectstore-ec delete mode 120000 ceph/qa/suites/multimds/basic/objectstore create mode 120000 ceph/qa/suites/multimds/basic/objectstore-ec delete mode 120000 ceph/qa/suites/multimds/thrash/objectstore create mode 120000 ceph/qa/suites/multimds/thrash/objectstore-ec delete mode 120000 ceph/qa/suites/multimds/verify/objectstore create mode 120000 ceph/qa/suites/multimds/verify/objectstore-ec delete mode 120000 ceph/qa/suites/rados/basic/d-require-luminous create mode 100644 ceph/qa/suites/rados/basic/d-require-luminous/at-end.yaml rename ceph/qa/suites/{upgrade/jewel-x/parallel/5-final-workload/+ => rados/basic/d-require-luminous/at-mkfs.yaml} (100%) create mode 100644 ceph/qa/suites/rados/mgr/tasks/dashboard.yaml create mode 100644 ceph/qa/suites/rados/mgr/tasks/module_selftest.yaml create mode 100644 ceph/qa/suites/rados/mgr/tasks/workunits.yaml rename ceph/qa/suites/{rest/basic/tasks => rados/rest}/rest_test.yaml (62%) create mode 100644 ceph/qa/suites/rados/singleton/all/max-pg-per-osd.from-mon.yaml create mode 100644 ceph/qa/suites/rados/singleton/all/max-pg-per-osd.from-primary.yaml create mode 100644 ceph/qa/suites/rados/singleton/all/max-pg-per-osd.from-replica.yaml create mode 100644 ceph/qa/suites/rados/singleton/all/recovery-preemption.yaml create mode 100644 ceph/qa/suites/rados/thrash/d-require-luminous/at-mkfs-balancer-crush-compat.yaml create mode 100644 ceph/qa/suites/rados/thrash/d-require-luminous/at-mkfs-balancer-upmap.yaml rename ceph/qa/suites/upgrade/{kraken-x/parallel/5-final-workload/+ => jewel-x/ceph-deploy/%} (100%) create mode 120000 ceph/qa/suites/upgrade/jewel-x/ceph-deploy/distros/centos_latest.yaml create mode 120000 ceph/qa/suites/upgrade/jewel-x/ceph-deploy/distros/ubuntu_latest.yaml create mode 100644 ceph/qa/suites/upgrade/jewel-x/ceph-deploy/jewel-luminous.yaml mode change 120000 => 100644 ceph/qa/suites/upgrade/jewel-x/parallel/4-luminous.yaml create mode 100644 ceph/qa/suites/upgrade/jewel-x/parallel/5-workload.yaml rename ceph/qa/suites/upgrade/{kraken-x/parallel/4-luminous-with-mgr.yaml => jewel-x/parallel/6-luminous-with-mgr.yaml} (100%) create mode 100644 ceph/qa/suites/upgrade/jewel-x/parallel/6.5-crush-compat.yaml create mode 100644 ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/+ rename ceph/qa/suites/upgrade/{kraken-x/parallel/5-final-workload => jewel-x/parallel/7-final-workload}/blogbench.yaml (74%) rename ceph/qa/suites/upgrade/{kraken-x/parallel/5-final-workload => jewel-x/parallel/7-final-workload}/rados-snaps-few-objects.yaml (88%) rename ceph/qa/suites/upgrade/{kraken-x/parallel/5-final-workload => jewel-x/parallel/7-final-workload}/rados_loadgenmix.yaml (74%) rename ceph/qa/suites/upgrade/jewel-x/parallel/{5-final-workload => 7-final-workload}/rados_mon_thrash.yaml (84%) rename ceph/qa/suites/upgrade/{kraken-x/parallel/5-final-workload => jewel-x/parallel/7-final-workload}/rbd_cls.yaml (69%) rename ceph/qa/suites/upgrade/{kraken-x/parallel/5-final-workload => jewel-x/parallel/7-final-workload}/rbd_import_export.yaml (76%) rename ceph/qa/suites/upgrade/{kraken-x/parallel/5-final-workload => jewel-x/parallel/7-final-workload}/rgw_swift.yaml (64%) create mode 120000 ceph/qa/suites/upgrade/jewel-x/parallel/8-jewel-workload.yaml create mode 120000 ceph/qa/suites/upgrade/jewel-x/stress-split/6.5-crush-compat.yaml create mode 100644 ceph/qa/suites/upgrade/kraken-x/ceph-deploy/kraken-luminous.yaml create mode 100644 ceph/qa/suites/upgrade/kraken-x/parallel/4-luminous.yaml create mode 100644 ceph/qa/suites/upgrade/kraken-x/parallel/5-workload.yaml create mode 120000 ceph/qa/suites/upgrade/kraken-x/parallel/6-luminous-with-mgr.yaml create mode 100644 ceph/qa/suites/upgrade/kraken-x/parallel/7-final-workload/+ rename ceph/qa/suites/upgrade/{jewel-x/parallel/5-final-workload => kraken-x/parallel/7-final-workload}/blogbench.yaml (100%) rename ceph/qa/suites/upgrade/{jewel-x/parallel/5-final-workload => kraken-x/parallel/7-final-workload}/rados-snaps-few-objects.yaml (100%) rename ceph/qa/suites/upgrade/{jewel-x/parallel/5-final-workload => kraken-x/parallel/7-final-workload}/rados_loadgenmix.yaml (100%) rename ceph/qa/suites/upgrade/kraken-x/parallel/{5-final-workload => 7-final-workload}/rados_mon_thrash.yaml (100%) rename ceph/qa/suites/upgrade/{jewel-x/parallel/5-final-workload => kraken-x/parallel/7-final-workload}/rbd_cls.yaml (100%) rename ceph/qa/suites/upgrade/{jewel-x/parallel/5-final-workload => kraken-x/parallel/7-final-workload}/rbd_import_export.yaml (100%) rename ceph/qa/suites/upgrade/{jewel-x/parallel/5-final-workload => kraken-x/parallel/7-final-workload}/rgw_swift.yaml (100%) create mode 100644 ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_import_export_no_upgrated.yaml rename ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/{rbd_import_export.yaml => rbd_import_export_upgrated.yaml} (65%) create mode 120000 ceph/qa/suites/upgrade/luminous-x/stress-split-erasure-code/1-ceph-install create mode 100644 ceph/qa/tasks/mgr/test_dashboard.py create mode 100644 ceph/qa/tasks/mgr/test_module_selftest.py create mode 100644 ceph/qa/tasks/osd_max_pg_per_osd.py create mode 100755 ceph/qa/workunits/cls/test_cls_journal.sh create mode 100755 ceph/qa/workunits/mgr/test_localpool.sh create mode 100644 ceph/src/90-ceph-osd.conf mode change 100755 => 100644 ceph/src/ceph-disk/ceph_disk/main.py create mode 100644 ceph/src/ceph-volume/ceph_volume/api/__init__.py rename ceph/src/ceph-volume/ceph_volume/{devices/lvm/api.py => api/lvm.py} (89%) create mode 100644 ceph/src/ceph-volume/ceph_volume/devices/lvm/listing.py create mode 100644 ceph/src/ceph-volume/ceph_volume/devices/lvm/zap.py create mode 100644 ceph/src/ceph-volume/ceph_volume/devices/simple/__init__.py create mode 100644 ceph/src/ceph-volume/ceph_volume/devices/simple/activate.py create mode 100644 ceph/src/ceph-volume/ceph_volume/devices/simple/main.py create mode 100644 ceph/src/ceph-volume/ceph_volume/devices/simple/scan.py create mode 100644 ceph/src/ceph-volume/ceph_volume/devices/simple/trigger.py rename ceph/src/ceph-volume/ceph_volume/tests/{devices/lvm/test_api.py => api/test_lvm.py} (84%) create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_listing.py create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/devices/simple/test_activate.py create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/devices/simple/test_scan.py create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/devices/simple/test_trigger.py create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/devices/test_zap.py delete mode 120000 ceph/src/ceph-volume/ceph_volume/tests/functional/centos7/create/Vagrantfile create mode 120000 ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/Vagrantfile create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/group_vars/all rename ceph/src/ceph-volume/ceph_volume/tests/functional/{centos7 => lvm/centos7/bluestore}/create/hosts (100%) create mode 120000 ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/setup.yml rename ceph/src/ceph-volume/ceph_volume/tests/functional/{centos7 => lvm/centos7/bluestore}/create/vagrant_variables.yml (100%) create mode 120000 ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/Vagrantfile rename ceph/src/ceph-volume/ceph_volume/tests/functional/{centos7 => lvm/centos7/filestore}/create/group_vars/all (94%) rename ceph/src/ceph-volume/ceph_volume/tests/functional/{xenial => lvm/centos7/filestore}/create/hosts (100%) create mode 120000 ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/setup.yml create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/vagrant_variables.yml create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/setup_partitions.yml create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini create mode 120000 ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/Vagrantfile create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/group_vars/all create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/hosts create mode 120000 ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/setup.yml create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/vagrant_variables.yml create mode 120000 ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/Vagrantfile rename ceph/src/ceph-volume/ceph_volume/tests/functional/{xenial => lvm/xenial/filestore}/create/group_vars/all (94%) create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/hosts create mode 120000 ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/setup.yml rename ceph/src/ceph-volume/ceph_volume/tests/functional/{xenial => lvm/xenial/filestore}/create/vagrant_variables.yml (100%) create mode 120000 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/Vagrantfile create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/group_vars/all create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/host_vars/osd0.yml create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/host_vars/osd1.yml create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/hosts create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/test.yml create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/vagrant_variables.yml create mode 120000 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/Vagrantfile create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/group_vars/all create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/host_vars/osd0.yml create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/host_vars/osd1.yml create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/hosts create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/test.yml create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/vagrant_variables.yml rename ceph/src/ceph-volume/ceph_volume/tests/functional/{ => simple}/tox.ini (68%) create mode 120000 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/Vagrantfile create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/group_vars/all create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/host_vars/osd0.yml create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/host_vars/osd1.yml create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/hosts create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/test.yml create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/vagrant_variables.yml create mode 120000 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/Vagrantfile create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/group_vars/all create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/host_vars/osd0.yml create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/host_vars/osd1.yml create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/hosts create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/test.yml create mode 100644 ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/vagrant_variables.yml delete mode 120000 ceph/src/ceph-volume/ceph_volume/tests/functional/xenial/create/Vagrantfile create mode 100644 ceph/src/mgr/ActivePyModule.cc rename ceph/src/mgr/{MgrPyModule.h => ActivePyModule.h} (70%) rename ceph/src/mgr/{PyModules.cc => ActivePyModules.cc} (61%) rename ceph/src/mgr/{PyModules.h => ActivePyModules.h} (66%) create mode 100644 ceph/src/mgr/BaseMgrModule.cc create mode 100644 ceph/src/mgr/BaseMgrModule.h create mode 100644 ceph/src/mgr/BaseMgrStandbyModule.cc create mode 100644 ceph/src/mgr/BaseMgrStandbyModule.h create mode 100644 ceph/src/mgr/Gil.cc delete mode 100644 ceph/src/mgr/MgrPyModule.cc create mode 100644 ceph/src/mgr/PyModuleRegistry.cc create mode 100644 ceph/src/mgr/PyModuleRegistry.h create mode 100644 ceph/src/mgr/PyModuleRunner.cc create mode 100644 ceph/src/mgr/PyModuleRunner.h create mode 100644 ceph/src/mgr/PyOSDMap.cc create mode 100644 ceph/src/mgr/PyOSDMap.h delete mode 100644 ceph/src/mgr/PyState.cc delete mode 100644 ceph/src/mgr/PyState.h create mode 100644 ceph/src/mgr/StandbyPyModules.cc create mode 100644 ceph/src/mgr/StandbyPyModules.h create mode 100644 ceph/src/pybind/mgr/balancer/__init__.py create mode 100644 ceph/src/pybind/mgr/balancer/module.py create mode 100644 ceph/src/pybind/mgr/dashboard/standby.html create mode 100644 ceph/src/pybind/mgr/influx/__init__.py create mode 100644 ceph/src/pybind/mgr/influx/module.py create mode 100644 ceph/src/pybind/mgr/localpool/__init__.py create mode 100644 ceph/src/pybind/mgr/localpool/module.py create mode 100644 ceph/src/pybind/mgr/selftest/__init__.py create mode 100644 ceph/src/pybind/mgr/selftest/module.py create mode 100644 ceph/src/test/cli/osdmaptool/upmap-out.t create mode 100644 ceph/src/test/cls_journal/CMakeLists.txt create mode 100644 ceph/src/test/librbd/operation/test_mock_TrimRequest.cc create mode 100644 ceph/src/test/librgw_file_marker.cc diff --git a/ceph/CMakeLists.txt b/ceph/CMakeLists.txt index c358e3e97..0d362d849 100644 --- a/ceph/CMakeLists.txt +++ b/ceph/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 2.8.11) project(ceph) -set(VERSION 12.2.1) +set(VERSION 12.2.2) if(POLICY CMP0046) # Tweak policies (this one disables "missing" dependency warning) diff --git a/ceph/COPYING b/ceph/COPYING index b7371e4f9..a87427936 100644 --- a/ceph/COPYING +++ b/ceph/COPYING @@ -145,8 +145,3 @@ Files: src/include/timegm.h Copyright (C) Copyright Howard Hinnant Copyright (C) Copyright 2010-2011 Vicente J. Botet Escriba License: Boost Software License, Version 1.0 - -Files: src/msg/async/AsyncConnection.cc, src/msg/simple/Pipe.cc (sigpipe suppression) - Copyright (C) 2010 Tomash Brechko. All rights reserved. - License: GPL3 - diff --git a/ceph/PendingReleaseNotes b/ceph/PendingReleaseNotes index 9ca48cdab..b46d1dce1 100644 --- a/ceph/PendingReleaseNotes +++ b/ceph/PendingReleaseNotes @@ -27,3 +27,14 @@ limit (5% by default). Limits by inode count are still supported using mds_cache_size. Setting mds_cache_size to 0 (the default) disables the inode limit. + +* The maximum number of PGs per OSD before the monitor issues a + warning has been reduced from 300 to 200 PGs. 200 is still twice + the generally recommended target of 100 PGs per OSD. This limit can + be adjusted via the ``mon_max_pg_per_osd`` option on the + monitors. The older ``mon_pg_warn_max_per_osd`` option has been removed. + +* Creating pools or adjusting pg_num will now fail if the change would + make the number of PGs per OSD exceed the configured + ``mon_max_pg_per_osd`` limit. The option can be adjusted if it + is really necessary to create a pool with more PGs. diff --git a/ceph/admin/doc-requirements.txt b/ceph/admin/doc-requirements.txt index aba92c28b..dc1411303 100644 --- a/ceph/admin/doc-requirements.txt +++ b/ceph/admin/doc-requirements.txt @@ -1,3 +1,3 @@ -Sphinx == 1.1.3 --e git+https://github.com/ceph/sphinx-ditaa.git#egg=sphinx-ditaa +Sphinx == 1.6.3 +-e git+https://github.com/ceph/sphinx-ditaa.git@py3#egg=sphinx-ditaa -e git+https://github.com/michaeljones/breathe#egg=breathe diff --git a/ceph/alpine/APKBUILD b/ceph/alpine/APKBUILD index 5fd0a9a24..0a0d4a472 100644 --- a/ceph/alpine/APKBUILD +++ b/ceph/alpine/APKBUILD @@ -1,7 +1,7 @@ # Contributor: John Coyle # Maintainer: John Coyle pkgname=ceph -pkgver=12.2.1 +pkgver=12.2.2 pkgrel=0 pkgdesc="Ceph is a distributed object store and file system" pkgusers="ceph" @@ -63,7 +63,7 @@ makedepends=" xmlstarlet yasm " -source="ceph-12.2.1.tar.bz2" +source="ceph-12.2.2.tar.bz2" subpackages=" $pkgname-base $pkgname-common @@ -116,7 +116,7 @@ _sysconfdir=/etc _udevrulesdir=/etc/udev/rules.d _python_sitelib=/usr/lib/python2.7/site-packages -builddir=$srcdir/ceph-12.2.1 +builddir=$srcdir/ceph-12.2.2 build() { export CEPH_BUILD_VIRTUALENV=$builddir diff --git a/ceph/ceph.spec b/ceph/ceph.spec index 00d09ee21..a4e5f8c94 100644 --- a/ceph/ceph.spec +++ b/ceph/ceph.spec @@ -61,13 +61,14 @@ # main package definition ################################################################################# Name: ceph -Version: 12.2.1 +Version: 12.2.2 Release: 0%{?dist} %if 0%{?fedora} || 0%{?rhel} Epoch: 2 %endif -# define %_epoch_prefix macro which will expand to the empty string if %epoch is undefined +# define _epoch_prefix macro which will expand to the empty string if epoch is +# undefined %global _epoch_prefix %{?epoch:%{epoch}:} Summary: User space components of the Ceph file system @@ -76,7 +77,7 @@ License: LGPL-2.1 and CC-BY-SA-1.0 and GPL-2.0 and BSL-1.0 and BSD-3-Clause and Group: System/Filesystems %endif URL: http://ceph.com/ -Source0: http://ceph.com/download/ceph-12.2.1.tar.bz2 +Source0: http://ceph.com/download/ceph-12.2.2.tar.bz2 %if 0%{?suse_version} %if 0%{?is_opensuse} ExclusiveArch: x86_64 aarch64 ppc64 ppc64le @@ -109,6 +110,7 @@ BuildRequires: python-werkzeug %if 0%{?suse_version} BuildRequires: python-CherryPy BuildRequires: python-Werkzeug +BuildRequires: python-numpy-devel %endif BuildRequires: python-pecan BuildRequires: socat @@ -773,7 +775,7 @@ python-rbd, python-rgw or python-cephfs instead. # common ################################################################################# %prep -%autosetup -p1 -n ceph-12.2.1 +%autosetup -p1 -n ceph-12.2.2 %build %if 0%{with cephfs_java} @@ -883,6 +885,7 @@ mkdir -p %{buildroot}%{_sbindir} install -m 0644 -D src/logrotate.conf %{buildroot}%{_sysconfdir}/logrotate.d/ceph chmod 0644 %{buildroot}%{_docdir}/ceph/sample.ceph.conf install -m 0644 -D COPYING %{buildroot}%{_docdir}/ceph/COPYING +install -m 0644 -D src/90-ceph-osd.conf %{buildroot}%{_sysctldir}/90-ceph-osd.conf # firewall templates and /sbin/mount.ceph symlink %if 0%{?suse_version} @@ -1412,12 +1415,14 @@ fi %{_udevrulesdir}/95-ceph-osd.rules %{_mandir}/man8/ceph-clsinfo.8* %{_mandir}/man8/ceph-osd.8* +%{_mandir}/man8/ceph-bluestore-tool.8* %if 0%{?rhel} && ! 0%{?centos} %attr(0755,-,-) %{_sysconfdir}/cron.hourly/subman %endif %{_unitdir}/ceph-osd@.service %{_unitdir}/ceph-osd.target %attr(750,ceph,ceph) %dir %{_localstatedir}/lib/ceph/osd +%config(noreplace) %{_sysctldir}/90-ceph-osd.conf %post osd %if 0%{?suse_version} @@ -1431,6 +1436,11 @@ fi if [ $1 -eq 1 ] ; then /usr/bin/systemctl start ceph-osd.target >/dev/null 2>&1 || : fi +%if 0%{?sysctl_apply} + %sysctl_apply 90-ceph-osd.conf +%else + /usr/lib/systemd/systemd-sysctl %{_sysctldir}/90-ceph-osd.conf > /dev/null 2>&1 || : +%endif %preun osd %if 0%{?suse_version} diff --git a/ceph/ceph.spec.in b/ceph/ceph.spec.in index b45c9feec..cc0830e60 100644 --- a/ceph/ceph.spec.in +++ b/ceph/ceph.spec.in @@ -67,7 +67,8 @@ Release: @RPM_RELEASE@%{?dist} Epoch: 2 %endif -# define %_epoch_prefix macro which will expand to the empty string if %epoch is undefined +# define _epoch_prefix macro which will expand to the empty string if epoch is +# undefined %global _epoch_prefix %{?epoch:%{epoch}:} Summary: User space components of the Ceph file system @@ -109,6 +110,7 @@ BuildRequires: python-werkzeug %if 0%{?suse_version} BuildRequires: python-CherryPy BuildRequires: python-Werkzeug +BuildRequires: python-numpy-devel %endif BuildRequires: python-pecan BuildRequires: socat @@ -883,6 +885,7 @@ mkdir -p %{buildroot}%{_sbindir} install -m 0644 -D src/logrotate.conf %{buildroot}%{_sysconfdir}/logrotate.d/ceph chmod 0644 %{buildroot}%{_docdir}/ceph/sample.ceph.conf install -m 0644 -D COPYING %{buildroot}%{_docdir}/ceph/COPYING +install -m 0644 -D src/90-ceph-osd.conf %{buildroot}%{_sysctldir}/90-ceph-osd.conf # firewall templates and /sbin/mount.ceph symlink %if 0%{?suse_version} @@ -1412,12 +1415,14 @@ fi %{_udevrulesdir}/95-ceph-osd.rules %{_mandir}/man8/ceph-clsinfo.8* %{_mandir}/man8/ceph-osd.8* +%{_mandir}/man8/ceph-bluestore-tool.8* %if 0%{?rhel} && ! 0%{?centos} %attr(0755,-,-) %{_sysconfdir}/cron.hourly/subman %endif %{_unitdir}/ceph-osd@.service %{_unitdir}/ceph-osd.target %attr(750,ceph,ceph) %dir %{_localstatedir}/lib/ceph/osd +%config(noreplace) %{_sysctldir}/90-ceph-osd.conf %post osd %if 0%{?suse_version} @@ -1431,6 +1436,11 @@ fi if [ $1 -eq 1 ] ; then /usr/bin/systemctl start ceph-osd.target >/dev/null 2>&1 || : fi +%if 0%{?sysctl_apply} + %sysctl_apply 90-ceph-osd.conf +%else + /usr/lib/systemd/systemd-sysctl %{_sysctldir}/90-ceph-osd.conf > /dev/null 2>&1 || : +%endif %preun osd %if 0%{?suse_version} diff --git a/ceph/debian/ceph-osd.install b/ceph/debian/ceph-osd.install index 262082cfd..87cd5011c 100644 --- a/ceph/debian/ceph-osd.install +++ b/ceph/debian/ceph-osd.install @@ -19,3 +19,5 @@ usr/share/man/man8/ceph-disk.8 usr/share/man/man8/ceph-volume.8 usr/share/man/man8/ceph-volume-systemd.8 usr/share/man/man8/ceph-osd.8 +usr/share/man/man8/ceph-bluestore-tool.8 +etc/sysctl.d/30-ceph-osd.conf diff --git a/ceph/debian/ceph-osd.postinst b/ceph/debian/ceph-osd.postinst index b642dfe34..5e44548fe 100644 --- a/ceph/debian/ceph-osd.postinst +++ b/ceph/debian/ceph-osd.postinst @@ -23,6 +23,7 @@ set -e case "$1" in configure) + [ -x /etc/init.d/procps ] && invoke-rc.d procps restart || : [ -x /sbin/start ] && start ceph-osd-all || : ;; abort-upgrade|abort-remove|abort-deconfigure) diff --git a/ceph/debian/changelog b/ceph/debian/changelog index f05243917..7597cb680 100644 --- a/ceph/debian/changelog +++ b/ceph/debian/changelog @@ -1,3 +1,9 @@ +ceph (12.2.2-1) stable; urgency=medium + + * New upstream release + + -- Ceph Release Team Thu, 30 Nov 2017 14:59:26 +0000 + ceph (12.2.1-1) stable; urgency=medium * New upstream release diff --git a/ceph/debian/rules b/ceph/debian/rules index 92bc0b587..857888f84 100755 --- a/ceph/debian/rules +++ b/ceph/debian/rules @@ -50,6 +50,7 @@ override_dh_auto_install: install -D -m 644 udev/95-ceph-osd.rules $(DESTDIR)/lib/udev/rules.d/95-ceph-osd.rules install -D -m 644 udev/60-ceph-by-parttypeuuid.rules $(DESTDIR)/lib/udev/rules.d/60-ceph-by-parttypeuuid.rules install -D -m 644 src/etc-rbdmap $(DESTDIR)/etc/ceph/rbdmap + install -D -m 644 src/90-ceph-osd.conf $(DESTDIR)/etc/sysctl.d/30-ceph-osd.conf # doc/changelog is a directory, which confuses dh_installchangelogs override_dh_installchangelogs: diff --git a/ceph/doc/ceph-volume/index.rst b/ceph/doc/ceph-volume/index.rst index d34e80294..5cf4778bb 100644 --- a/ceph/doc/ceph-volume/index.rst +++ b/ceph/doc/ceph-volume/index.rst @@ -3,19 +3,46 @@ ceph-volume =========== Deploy OSDs with different device technologies like lvm or physical disks using -pluggable tools (:doc:`lvm/index` itself is treated like a plugin). It tries to -follow the workflow of ``ceph-disk`` for deploying OSDs, with a predictable, -and robust way of preparing, activating, and starting OSDs. +pluggable tools (:doc:`lvm/index` itself is treated like a plugin) and trying to +follow a predictable, and robust way of preparing, activating, and starting OSDs. :ref:`Overview ` | :ref:`Plugin Guide ` | **Command Line Subcommands** -Although currently there is support for ``lvm``, the plan is to support other -technologies, including plain disks. +There is currently support for ``lvm``, and plain disks (with GPT partitions) +that may have been deployed with ``ceph-disk``. * :ref:`ceph-volume-lvm` +* :ref:`ceph-volume-simple` + + +Migrating +--------- +Starting on Ceph version 12.2.2, ``ceph-disk`` is deprecated. Deprecation +warnings will show up that will link to this page. It is strongly suggested +that users start consuming ``ceph-volume``. + +New deployments +^^^^^^^^^^^^^^^ +For new deployments, :ref:`ceph-volume-lvm` is recommended, it can use any +logical volume as input for data OSDs, or it can setup a minimal/naive logical +volume from a device. + +Existing OSDs +^^^^^^^^^^^^^ +If the cluster has OSDs that were provisioned with ``ceph-disk``, then +``ceph-volume`` can take over the management of these with +:ref:`ceph-volume-simple`. A scan is done on the data device or OSD directory, +and ``ceph-disk`` is fully disabled. + +Encrypted OSDs +^^^^^^^^^^^^^^ +If using encryption with OSDs, there is currently no support in ``ceph-volume`` +for this scenario (although support for this is coming soon). In this case, it +is OK to continue to use ``ceph-disk`` until ``ceph-volume`` fully supports it. +This page will be updated when that happens. .. toctree:: :hidden: @@ -23,8 +50,15 @@ technologies, including plain disks. :caption: Contents: intro + systemd lvm/index lvm/activate lvm/prepare lvm/scan lvm/systemd + lvm/list + lvm/zap + simple/index + simple/activate + simple/scan + simple/systemd diff --git a/ceph/doc/ceph-volume/lvm/activate.rst b/ceph/doc/ceph-volume/lvm/activate.rst index b9f30d69f..956a62a62 100644 --- a/ceph/doc/ceph-volume/lvm/activate.rst +++ b/ceph/doc/ceph-volume/lvm/activate.rst @@ -17,7 +17,7 @@ New OSDs To activate newly prepared OSDs both the :term:`OSD id` and :term:`OSD uuid` need to be supplied. For example:: - ceph-volume activate --filestore 0 0263644D-0BF1-4D6D-BC34-28BD98AE3BC8 + ceph-volume lvm activate --bluestore 0 0263644D-0BF1-4D6D-BC34-28BD98AE3BC8 .. note:: The UUID is stored in the ``osd_fsid`` file in the OSD path, which is generated when :ref:`ceph-volume-lvm-prepare` is used. @@ -46,7 +46,7 @@ For example:: Would start the discovery process for the OSD with an id of ``0`` and a UUID of ``8715BEB4-15C5-49DE-BA6F-401086EC7B41``. -.. note:: for more details on the systemd workflow see :ref:`ceph-volume-systemd` +.. note:: for more details on the systemd workflow see :ref:`ceph-volume-lvm-systemd` The systemd unit will look for the matching OSD device, and by looking at its :term:`LVM tags` will proceed to: @@ -58,6 +58,9 @@ The systemd unit will look for the matching OSD device, and by looking at its # start the ``ceph-osd@0`` systemd unit +.. note:: The system infers the objectstore type (filestore or bluestore) by + inspecting the LVM tags applied to the OSD devices + Existing OSDs ------------- For exsiting OSDs that have been deployed with different tooling, the only way @@ -66,7 +69,18 @@ See :ref:`ceph-volume-lvm-existing-osds` for details on how to proceed. Summary ------- -To recap the ``activate`` process: +To recap the ``activate`` process for :term:`bluestore`: + +#. require both :term:`OSD id` and :term:`OSD uuid` +#. enable the system unit with matching id and uuid +#. Create the ``tmpfs`` mount at the OSD directory in + ``/var/lib/ceph/osd/$cluster-$id/`` +#. Recreate all the files needed with ``ceph-bluestore-tool prime-osd-dir`` by + pointing it to the OSD ``block`` device. +#. the systemd unit will ensure all devices are ready and linked +#. the matching ``ceph-osd`` systemd unit will get started + +And for :term:`filestore`: #. require both :term:`OSD id` and :term:`OSD uuid` #. enable the system unit with matching id and uuid diff --git a/ceph/doc/ceph-volume/lvm/create.rst b/ceph/doc/ceph-volume/lvm/create.rst new file mode 100644 index 000000000..c90d1f6fa --- /dev/null +++ b/ceph/doc/ceph-volume/lvm/create.rst @@ -0,0 +1,24 @@ +.. _ceph-volume-lvm-create: + +``create`` +=========== +This subcommand wraps the two-step process to provision a new osd (calling +``prepare`` first and then ``activate``) into a single +one. The reason to prefer ``prepare`` and then ``activate`` is to gradually +introduce new OSDs into a cluster, and avoiding large amounts of data being +rebalanced. + +The single-call process unifies exactly what :ref:`ceph-volume-lvm-prepare` and +:ref:`ceph-volume-lvm-activate` do, with the convenience of doing it all at +once. + +There is nothing different to the process except the OSD will become up and in +immediately after completion. + +The backing objectstore can be specified with: + +* :ref:`--filestore ` +* :ref:`--bluestore ` + +All command line flags and options are the same as ``ceph-volume lvm prepare``. +Please refer to :ref:`ceph-volume-lvm-prepare` for details. diff --git a/ceph/doc/ceph-volume/lvm/index.rst b/ceph/doc/ceph-volume/lvm/index.rst index 5c1ef0117..9a2191fb5 100644 --- a/ceph/doc/ceph-volume/lvm/index.rst +++ b/ceph/doc/ceph-volume/lvm/index.rst @@ -11,6 +11,10 @@ Implements the functionality needed to deploy OSDs from the ``lvm`` subcommand: * :ref:`ceph-volume-lvm-activate` +* :ref:`ceph-volume-lvm-create` + +* :ref:`ceph-volume-lvm-list` + .. not yet implemented .. * :ref:`ceph-volume-lvm-scan` @@ -20,5 +24,5 @@ There are other aspects of the ``lvm`` subcommand that are internal and not exposed to the user, these sections explain how these pieces work together, clarifying the workflows of the tool. -:ref:`Systemd Units ` | +:ref:`Systemd Units ` | :ref:`lvm ` diff --git a/ceph/doc/ceph-volume/lvm/list.rst b/ceph/doc/ceph-volume/lvm/list.rst new file mode 100644 index 000000000..19e06000b --- /dev/null +++ b/ceph/doc/ceph-volume/lvm/list.rst @@ -0,0 +1,173 @@ +.. _ceph-volume-lvm-list: + +``list`` +======== +This subcommand will list any devices (logical and physical) that may be +associated with a Ceph cluster, as long as they contain enough metadata to +allow for that discovery. + +Output is grouped by the OSD ID associated with the devices, and unlike +``ceph-disk`` it does not provide any information for devices that aren't +associated with Ceph. + +Command line options: + +* ``--format`` Allows a ``json`` or ``pretty`` value. Defaults to ``pretty`` + which will group the device information in a human-readable format. + +Full Reporting +-------------- +When no positional arguments are used, a full reporting will be presented. This +means that all devices and logical volumes found in the system will be +displayed. + +Full ``pretty`` reporting for two OSDs, one with a lv as a journal, and another +one with a physical device may look similar to:: + + # ceph-volume lvm list + + + ====== osd.1 ======= + + [journal] /dev/journals/journal1 + + journal uuid C65n7d-B1gy-cqX3-vZKY-ZoE0-IEYM-HnIJzs + osd id 1 + cluster fsid ce454d91-d748-4751-a318-ff7f7aa18ffd + type journal + osd fsid 661b24f8-e062-482b-8110-826ffe7f13fa + data uuid SlEgHe-jX1H-QBQk-Sce0-RUls-8KlY-g8HgcZ + journal device /dev/journals/journal1 + data device /dev/test_group/data-lv2 + + [data] /dev/test_group/data-lv2 + + journal uuid C65n7d-B1gy-cqX3-vZKY-ZoE0-IEYM-HnIJzs + osd id 1 + cluster fsid ce454d91-d748-4751-a318-ff7f7aa18ffd + type data + osd fsid 661b24f8-e062-482b-8110-826ffe7f13fa + data uuid SlEgHe-jX1H-QBQk-Sce0-RUls-8KlY-g8HgcZ + journal device /dev/journals/journal1 + data device /dev/test_group/data-lv2 + + ====== osd.0 ======= + + [data] /dev/test_group/data-lv1 + + journal uuid cd72bd28-002a-48da-bdf6-d5b993e84f3f + osd id 0 + cluster fsid ce454d91-d748-4751-a318-ff7f7aa18ffd + type data + osd fsid 943949f0-ce37-47ca-a33c-3413d46ee9ec + data uuid TUpfel-Q5ZT-eFph-bdGW-SiNW-l0ag-f5kh00 + journal device /dev/sdd1 + data device /dev/test_group/data-lv1 + + [journal] /dev/sdd1 + + PARTUUID cd72bd28-002a-48da-bdf6-d5b993e84f3f + +.. note:: Tags are displayed in a readable format. The ``osd id`` key is stored + as a ``ceph.osd_id`` tag. For more information on lvm tag conventions + see :ref:`ceph-volume-lvm-tag-api` + +Single Reporting +---------------- +Single reporting can consume both devices and logical volumes as input +(positional parameters). For logical volumes, it is required to use the group +name as well as the logical volume name. + +For example the ``data-lv2`` logical volume, in the ``test_group`` volume group +can be listed in the following way:: + + # ceph-volume lvm list test_group/data-lv2 + + + ====== osd.1 ======= + + [data] /dev/test_group/data-lv2 + + journal uuid C65n7d-B1gy-cqX3-vZKY-ZoE0-IEYM-HnIJzs + osd id 1 + cluster fsid ce454d91-d748-4751-a318-ff7f7aa18ffd + type data + osd fsid 661b24f8-e062-482b-8110-826ffe7f13fa + data uuid SlEgHe-jX1H-QBQk-Sce0-RUls-8KlY-g8HgcZ + journal device /dev/journals/journal1 + data device /dev/test_group/data-lv2 + + +.. note:: Tags are displayed in a readable format. The ``osd id`` key is stored + as a ``ceph.osd_id`` tag. For more information on lvm tag conventions + see :ref:`ceph-volume-lvm-tag-api` + + +For plain disks, the full path to the device is required. For example, for +a device like ``/dev/sdd1`` it can look like:: + + + # ceph-volume lvm list /dev/sdd1 + + + ====== osd.0 ======= + + [journal] /dev/sdd1 + + PARTUUID cd72bd28-002a-48da-bdf6-d5b993e84f3f + + + +``json`` output +--------------- +All output using ``--format=json`` will show everything the system has stored +as metadata for the devices, including tags. + +No changes for readability are done with ``json`` reporting, and all +information is presented as-is. Full output as well as single devices can be +listed. + +For brevity, this is how a single logical volume would look with ``json`` +output (note how tags aren't modified):: + + # ceph-volume lvm list --format=json test_group/data-lv1 + { + "0": [ + { + "lv_name": "data-lv1", + "lv_path": "/dev/test_group/data-lv1", + "lv_tags": "ceph.cluster_fsid=ce454d91-d748-4751-a318-ff7f7aa18ffd,ceph.data_device=/dev/test_group/data-lv1,ceph.data_uuid=TUpfel-Q5ZT-eFph-bdGW-SiNW-l0ag-f5kh00,ceph.journal_device=/dev/sdd1,ceph.journal_uuid=cd72bd28-002a-48da-bdf6-d5b993e84f3f,ceph.osd_fsid=943949f0-ce37-47ca-a33c-3413d46ee9ec,ceph.osd_id=0,ceph.type=data", + "lv_uuid": "TUpfel-Q5ZT-eFph-bdGW-SiNW-l0ag-f5kh00", + "name": "data-lv1", + "path": "/dev/test_group/data-lv1", + "tags": { + "ceph.cluster_fsid": "ce454d91-d748-4751-a318-ff7f7aa18ffd", + "ceph.data_device": "/dev/test_group/data-lv1", + "ceph.data_uuid": "TUpfel-Q5ZT-eFph-bdGW-SiNW-l0ag-f5kh00", + "ceph.journal_device": "/dev/sdd1", + "ceph.journal_uuid": "cd72bd28-002a-48da-bdf6-d5b993e84f3f", + "ceph.osd_fsid": "943949f0-ce37-47ca-a33c-3413d46ee9ec", + "ceph.osd_id": "0", + "ceph.type": "data" + }, + "type": "data", + "vg_name": "test_group" + } + ] + } + + +Synchronized information +------------------------ +Before any listing type, the lvm API is queried to ensure that physical devices +that may be in use haven't changed naming. It is possible that non-persistent +devices like ``/dev/sda1`` could change to ``/dev/sdb1``. + +The detection is possible because the ``PARTUUID`` is stored as part of the +metadata in the logical volume for the data lv. Even in the case of a journal +that is a physical device, this information is still stored on the data logical +volume associated with it. + +If the name is no longer the same (as reported by ``blkid`` when using the +``PARTUUID``), the tag will get updated and the report will use the newly +refreshed information. diff --git a/ceph/doc/ceph-volume/lvm/prepare.rst b/ceph/doc/ceph-volume/lvm/prepare.rst index add0f185d..27ebb55d7 100644 --- a/ceph/doc/ceph-volume/lvm/prepare.rst +++ b/ceph/doc/ceph-volume/lvm/prepare.rst @@ -2,10 +2,11 @@ ``prepare`` =========== -This subcommand allows a :term:`filestore` setup (:term:`bluestore` support is -planned) and currently consumes only logical volumes for both the data and -journal. It will not create or modify the logical volumes except for adding -extra metadata. +This subcommand allows a :term:`filestore` or :term:`bluestore` setup. It is +recommended to pre-provision a logical volume before using it with +``ceph-volume lvm``. + +Logical volumes are not altered except for adding extra metadata. .. note:: This is part of a two step process to deploy an OSD. If looking for a single-call way, please see :ref:`ceph-volume-lvm-create` @@ -23,28 +24,46 @@ the back end can be specified with: * :ref:`--filestore ` -* ``--bluestore`` - -.. when available, this will need to be updated to: -.. * :ref:`--bluestore ` +* :ref:`--bluestore ` .. _ceph-volume-lvm-prepare_filestore: ``filestore`` ------------- -This is the default OSD backend and allows preparation of logical volumes for -a :term:`filestore` OSD. +This is the OSD backend that allows preparation of logical volumes for +a :term:`filestore` objectstore OSD. -The process is *very* strict, it requires two logical volumes that are ready to -be used. No special preparation is needed for these volumes other than -following the minimum size requirements for data and journal. +It can use a logical volume for the OSD data and a partitioned physical device +or logical volume for the journal. No special preparation is needed for these +volumes other than following the minimum size requirements for data and +journal. The API call looks like:: ceph-volume prepare --filestore --data data --journal journal -The journal *must* be a logical volume, just like the data volume, and that -argument is always required even if both live under the same group. +There is flexibility to use a raw device or partition as well for ``--data`` +that will be converted to a logical volume. This is not ideal in all situations +since ``ceph-volume`` is just going to create a unique volume group and +a logical volume from that device. + +When using logical volumes for ``--data``, the value *must* be a volume group +name and a logical volume name separated by a ``/``. Since logical volume names +are not enforced for uniqueness, this prevents using the wrong volume. The +``--journal`` can be either a logical volume *or* a partition. + +When using a partition, it *must* contain a ``PARTUUID`` discoverable by +``blkid``, so that it can later be identified correctly regardless of the +device name (or path). + +When using a partition, this is how it would look for ``/dev/sdc1``:: + + ceph-volume prepare --filestore --data volume_group/lv_name --journal /dev/sdc1 + +For a logical volume, just like for ``--data``, a volume group and logical +volume name are required:: + + ceph-volume prepare --filestore --data volume_group/lv_name --journal volume_group/journal_lv A generated uuid is used to ask the cluster for a new OSD. These two pieces are crucial for identifying an OSD and will later be used throughout the @@ -108,32 +127,109 @@ later be started (for detailed metadata description see :ref:`ceph-volume-lvm-ta ``bluestore`` ------------- -This subcommand is planned but not currently implemented. +The :term:`bluestore` objectstore is the default for new OSDs. It offers a bit +more flexibility for devices. Bluestore supports the following configurations: + +* A block device, a block.wal, and a block.db device +* A block device and a block.wal device +* A block device and a block.db device +* A single block device + +It can accept a whole device (or partition), or a logical volume for ``block``. +If a physical device is provided it will then be turned into a logical volume. +This allows a simpler approach at using LVM but at the cost of flexibility: +there are no options or configurations to change how the LV is created. + +The ``block`` is specified with the ``--data`` flag, and in its simplest use +case it looks like:: + + ceph-volume lvm prepare --bluestore --data vg/lv + +A raw device can be specified in the same way:: + + ceph-volume lvm prepare --bluestore --data /path/to/device + + +If a ``block.db`` or a ``block.wal`` is needed (they are optional for +bluestore) they can be specified with ``--block.db`` and ``--block.wal`` +accordingly. These can be a physical device (they **must** be a partition) or +a logical volume. + +For both ``block.db`` and ``block.wal`` partitions aren't made logical volumes +because they can be used as-is. Logical Volumes are also allowed. + +While creating the OSD directory, the process will use a ``tmpfs`` mount to +place all the files needed for the OSD. These files are initially created by +``ceph-osd --mkfs`` and are fully ephemeral. + +A symlink is always created for the ``block`` device, and optionally for +``block.db`` and ``block.wal``. For a cluster with a default name, and an OSD +id of 0, the directory could look like:: + + # ls -l /var/lib/ceph/osd/ceph-0 + lrwxrwxrwx. 1 ceph ceph 93 Oct 20 13:05 block -> /dev/ceph-be2b6fbd-bcf2-4c51-b35d-a35a162a02f0/osd-block-25cf0a05-2bc6-44ef-9137-79d65bd7ad62 + lrwxrwxrwx. 1 ceph ceph 93 Oct 20 13:05 block.db -> /dev/sda1 + lrwxrwxrwx. 1 ceph ceph 93 Oct 20 13:05 block.wal -> /dev/ceph/osd-wal-0 + -rw-------. 1 ceph ceph 37 Oct 20 13:05 ceph_fsid + -rw-------. 1 ceph ceph 37 Oct 20 13:05 fsid + -rw-------. 1 ceph ceph 55 Oct 20 13:05 keyring + -rw-------. 1 ceph ceph 6 Oct 20 13:05 ready + -rw-------. 1 ceph ceph 10 Oct 20 13:05 type + -rw-------. 1 ceph ceph 2 Oct 20 13:05 whoami + +In the above case, a device was used for ``block`` so ``ceph-volume`` create +a volume group and a logical volume using the following convention: + +* volume group name: ``ceph-{cluster fsid}`` or if the vg exists already + ``ceph-{random uuid}`` + +* logical volume name: ``osd-block-{osd_fsid}`` Storing metadata ---------------- -The following tags will get applied as part of the prepartion process -regardless of the type of volume (journal or data) and also regardless of the -OSD backend: +The following tags will get applied as part of the preparation process +regardless of the type of volume (journal or data) or OSD objectstore: * ``cluster_fsid`` -* ``data_device`` -* ``journal_device`` * ``encrypted`` * ``osd_fsid`` * ``osd_id`` -* ``block`` -* ``db`` -* ``wal`` -* ``lockbox_device`` + +For :term:`filestore` these tags will be added: + +* ``journal_device`` +* ``journal_uuid`` + +For :term:`bluestore` these tags will be added: + +* ``block_device`` +* ``block_uuid`` +* ``db_device`` +* ``db_uuid`` +* ``wal_device`` +* ``wal_uuid`` .. note:: For the complete lvm tag conventions see :ref:`ceph-volume-lvm-tag-api` Summary ------- -To recap the ``prepare`` process: +To recap the ``prepare`` process for :term:`bluestore`: + +#. Accept a logical volume for block or a raw device (that will get converted + to an lv) +#. Accept partitions or logical volumes for ``block.wal`` or ``block.db`` +#. Generate a UUID for the OSD +#. Ask the monitor get an OSD ID reusing the generated UUID +#. OSD data directory is created on a tmpfs mount. +#. ``block``, ``block.wal``, and ``block.db`` are symlinked if defined. +#. monmap is fetched for activation +#. Data directory is populated by ``ceph-osd`` +#. Logical Volumes are are assigned all the Ceph metadata using lvm tags + + +And the ``prepare`` process for :term:`filestore`: #. Accept only logical volumes for data and journal (both required) #. Generate a UUID for the OSD diff --git a/ceph/doc/ceph-volume/lvm/systemd.rst b/ceph/doc/ceph-volume/lvm/systemd.rst index 7162e0433..30260de7e 100644 --- a/ceph/doc/ceph-volume/lvm/systemd.rst +++ b/ceph/doc/ceph-volume/lvm/systemd.rst @@ -1,31 +1,7 @@ -.. _ceph-volume-systemd: +.. _ceph-volume-lvm-systemd: systemd ======= -As part of the :ref:`ceph-volume-lvm-activate` process, a few systemd units will get enabled -that will use the OSD id and uuid as part of their name. These units will be -run when the system boots, and will proceed to activate their corresponding -volumes. - -The API for activation requires both the :term:`OSD id` and :term:`OSD uuid`, -which get persisted by systemd. Internally, the activation process enables the -systemd unit using the following convention:: - - ceph-volume@- - -Where ``type`` is the sub-command used to parse the extra metadata, and ``extra -metadata`` is any additional information needed by the sub-command to be able -to activate the OSD. For example an OSD with an ID of 0, for the ``lvm`` -sub-command would look like:: - - systemctl enable ceph-volume@lvm-0-0A3E1ED2-DA8A-4F0E-AA95-61DEC71768D6 - - -Process -------- -The systemd unit is a :term:`systemd oneshot` service, meant to start at boot after the -local filesystem is ready to be used. - Upon startup, it will identify the logical volume using :term:`LVM tags`, finding a matching ID and later ensuring it is the right one with the :term:`OSD uuid`. @@ -41,6 +17,12 @@ be mounted at:: /var/lib/ceph/osd/ceph-0 + Once that process is complete, a call will be made to start the OSD:: systemctl start ceph-osd@0 + +The systemd portion of this process is handled by the ``ceph-volume lvm +trigger`` sub-command, which is only in charge of parsing metadata coming from +systemd and startup, and then dispatching to ``ceph-volume lvm activate`` which +would proceed with activation. diff --git a/ceph/doc/ceph-volume/lvm/zap.rst b/ceph/doc/ceph-volume/lvm/zap.rst new file mode 100644 index 000000000..8d42a9089 --- /dev/null +++ b/ceph/doc/ceph-volume/lvm/zap.rst @@ -0,0 +1,19 @@ +.. _ceph-volume-lvm-zap: + +``zap`` +======= + +This subcommand is used to zap lvs or partitions that have been used +by ceph OSDs so that they may be reused. If given a path to a logical +volume it must be in the format of vg/lv. Any filesystems present +on the given lv or partition will be removed and all data will be purged. + +.. note:: The lv or partition will be kept intact. + +Zapping a logical volume:: + + ceph-volume lvm zap {vg name/lv name} + +Zapping a partition:: + + ceph-volume lvm zap /dev/sdc1 diff --git a/ceph/doc/ceph-volume/simple/activate.rst b/ceph/doc/ceph-volume/simple/activate.rst new file mode 100644 index 000000000..edbb1e3f8 --- /dev/null +++ b/ceph/doc/ceph-volume/simple/activate.rst @@ -0,0 +1,80 @@ +.. _ceph-volume-simple-activate: + +``activate`` +============ +Once :ref:`ceph-volume-simple-scan` has been completed, and all the metadata +captured for an OSD has been persisted to ``/etc/ceph/osd/{id}-{uuid}.json`` +the OSD is now ready to get "activated". + +This activation process **disables** all ``ceph-disk`` systemd units by masking +them, to prevent the UDEV/ceph-disk interaction that will attempt to start them +up at boot time. + +The disabling of ``ceph-disk`` units is done only when calling ``ceph-volume +simple activate`` directly, but is is avoided when being called by systemd when +the system is booting up. + +The activation process requires using both the :term:`OSD id` and :term:`OSD uuid` +To activate parsed OSDs:: + + ceph-volume simple activate 0 6cc43680-4f6e-4feb-92ff-9c7ba204120e + +The above command will assume that a JSON configuration will be found in:: + + /etc/ceph/osd/0-6cc43680-4f6e-4feb-92ff-9c7ba204120e.json + +Alternatively, using a path to a JSON file directly is also possible:: + + ceph-volume simple activate --file /etc/ceph/osd/0-6cc43680-4f6e-4feb-92ff-9c7ba204120e.json + +requiring uuids +^^^^^^^^^^^^^^^ +The :term:`OSD uuid` is being required as an extra step to ensure that the +right OSD is being activated. It is entirely possible that a previous OSD with +the same id exists and would end up activating the incorrect one. + + +Discovery +--------- +With OSDs previously scanned by ``ceph-volume``, a *discovery* process is +performed using ``blkid`` and ``lvm``. There is currently support only for +devices with GPT partitions and LVM logical volumes. + +The GPT partitions will have a ``PARTUUID`` that can be queried by calling out +to ``blkid``, and the logical volumes will have a ``lv_uuid`` that can be +queried against ``lvs`` (the LVM tool to list logical volumes). + +This discovery process ensures that devices can be correctly detected even if +they are repurposed into another system or if their name changes (as in the +case of non-persisting names like ``/dev/sda1``) + +The JSON configuration file used to map what devices go to what OSD will then +coordinate the mounting and symlinking as part of activation. + +To ensure that the symlinks are always correct, if they exist in the OSD +directory, the symlinks will be re-done. + +A systemd unit will capture the :term:`OSD id` and :term:`OSD uuid` and +persist it. Internally, the activation will enable it like:: + + systemctl enable ceph-volume@simple-$id-$uuid + +For example:: + + systemctl enable ceph-volume@simple-0-8715BEB4-15C5-49DE-BA6F-401086EC7B41 + +Would start the discovery process for the OSD with an id of ``0`` and a UUID of +``8715BEB4-15C5-49DE-BA6F-401086EC7B41``. + + +The systemd process will call out to activate passing the information needed to +identify the OSD and its devices, and it will proceed to: + +# mount the device in the corresponding location (by convention this is + ``/var/lib/ceph/osd/-/``) + +# ensure that all required devices are ready for that OSD and properly linked, +regardless of objectstore used (filestore or bluestore). The symbolic link will +**always** be re-done to ensure that the correct device is linked. + +# start the ``ceph-osd@0`` systemd unit diff --git a/ceph/doc/ceph-volume/simple/index.rst b/ceph/doc/ceph-volume/simple/index.rst new file mode 100644 index 000000000..6f2534a73 --- /dev/null +++ b/ceph/doc/ceph-volume/simple/index.rst @@ -0,0 +1,19 @@ +.. _ceph-volume-simple: + +``simple`` +========== +Implements the functionality needed to manage OSDs from the ``simple`` subcommand: +``ceph-volume simple`` + +**Command Line Subcommands** + +* :ref:`ceph-volume-simple-scan` + +* :ref:`ceph-volume-simple-activate` + +* :ref:`ceph-volume-simple-systemd` + + +By *taking over* management, it disables all ``ceph-disk`` systemd units used +to trigger devices at startup, relying on basic (customizable) JSON +configuration and systemd for starting up OSDs. diff --git a/ceph/doc/ceph-volume/simple/scan.rst b/ceph/doc/ceph-volume/simple/scan.rst new file mode 100644 index 000000000..afeddabb8 --- /dev/null +++ b/ceph/doc/ceph-volume/simple/scan.rst @@ -0,0 +1,158 @@ +.. _ceph-volume-simple-scan: + +``scan`` +======== +Scanning allows to capture any important details from an already-deployed OSD +so that ``ceph-volume`` can manage it without the need of any other startup +workflows or tools (like ``udev`` or ``ceph-disk``). + +The command has the ability to inspect a running OSD, by inspecting the +directory where the OSD data is stored, or by consuming the data partition. + +Once scanned, information will (by default) persist the metadata as JSON in +a file in ``/etc/ceph/osd``. This ``JSON`` file will use the naming convention +of: ``{OSD ID}-{OSD FSID}.json``. An OSD with an id of 1, and an FSID like +``86ebd829-1405-43d3-8fd6-4cbc9b6ecf96`` the absolute path of the file would +be:: + + /etc/ceph/osd/1-86ebd829-1405-43d3-8fd6-4cbc9b6ecf96.json + +The ``scan`` subcommand will refuse to write to this file if it already exists. +If overwriting the contents is needed, the ``--force`` flag must be used:: + + ceph-volume simple scan --force {path} + +If there is no need to persist the ``JSON`` metadata, there is support to send +the contents to ``stdout`` (no file will be written):: + + ceph-volume simple scan --stdout {path} + + +.. _ceph-volume-simple-scan-directory: + +Directory scan +-------------- +The directory scan will capture OSD file contents from interesting files. There +are a few files that must exist in order to have a successful scan: + +* ``ceph_fsid`` +* ``fsid`` +* ``keyring`` +* ``ready`` +* ``type`` +* ``whoami`` + +In the case of any other file, as long as it is not a binary or a directory, it +will also get captured and persisted as part of the JSON object. + +The convention for the keys in the JSON object is that any file name will be +a key, and its contents will be its value. If the contents are a single line +(like in the case of the ``whoami``) the contents are trimmed, and the newline +is dropped. For example with an OSD with an id of 1, this is how the JSON entry +would look like:: + + "whoami": "1", + +For files that may have more than one line, the contents are left as-is, for +example, a ``keyring`` could look like this:: + + "keyring": "[osd.1]\n\tkey = AQBBJ/dZp57NIBAAtnuQS9WOS0hnLVe0rZnE6Q==\n", + +For a directory like ``/var/lib/ceph/osd/ceph-1``, the command could look +like:: + + ceph-volume simple scan /var/lib/ceph/osd/ceph1 + + +.. note:: There is no support for encrypted OSDs + + +.. _ceph-volume-simple-scan-device: + +Device scan +----------- +When an OSD directory is not available (OSD is not running, or device is not +mounted) the ``scan`` command is able to introspect the device to capture +required data. Just like :ref:`ceph-volume-simple-scan-directory`, it would +still require a few files present. This means that the device to be scanned +**must be** the data partition of the OSD. + +As long as the data partition of the OSD is being passed in as an argument, the +sub-command can scan its contents. + +In the case where the device is already mounted, the tool can detect this +scenario and capture file contents from that directory. + +If the device is not mounted, a temporary directory will be created, and the +device will be mounted temporarily just for scanning the contents. Once +contents are scanned, the device will be unmounted. + +For a device like ``/dev/sda1`` which **must** be a data partition, the command +could look like:: + + ceph-volume simple scan /dev/sda1 + + +.. note:: There is no support for encrypted OSDs + + +.. _ceph-volume-simple-scan-json: + +``JSON`` contents +----------------- +The contents of the JSON object is very simple. The scan not only will persist +information from the special OSD files and their contents, but will also +validate paths and device UUIDs. Unlike what ``ceph-disk`` would do, by storing +them in ``{device type}_uuid`` files, the tool will persist them as part of the +device type key. + +For example, a ``block.db`` device would look something like:: + + "block.db": { + "path": "/dev/disk/by-partuuid/6cc43680-4f6e-4feb-92ff-9c7ba204120e", + "uuid": "6cc43680-4f6e-4feb-92ff-9c7ba204120e" + }, + +But it will also persist the ``ceph-disk`` special file generated, like so:: + + "block.db_uuid": "6cc43680-4f6e-4feb-92ff-9c7ba204120e", + +This duplication is in place because the tool is trying to ensure the +following: + +# Support OSDs that may not have ceph-disk special files +# Check the most up-to-date information on the device, by querying against LVM +and ``blkid`` +# Support both logical volumes and GPT devices + +This is a sample ``JSON`` metadata, from an OSD that is using ``bluestore``:: + + { + "active": "ok", + "block": { + "path": "/dev/disk/by-partuuid/40fd0a64-caa5-43a3-9717-1836ac661a12", + "uuid": "40fd0a64-caa5-43a3-9717-1836ac661a12" + }, + "block.db": { + "path": "/dev/disk/by-partuuid/6cc43680-4f6e-4feb-92ff-9c7ba204120e", + "uuid": "6cc43680-4f6e-4feb-92ff-9c7ba204120e" + }, + "block.db_uuid": "6cc43680-4f6e-4feb-92ff-9c7ba204120e", + "block_uuid": "40fd0a64-caa5-43a3-9717-1836ac661a12", + "bluefs": "1", + "ceph_fsid": "c92fc9eb-0610-4363-aafc-81ddf70aaf1b", + "cluster_name": "ceph", + "data": { + "path": "/dev/sdr1", + "uuid": "86ebd829-1405-43d3-8fd6-4cbc9b6ecf96" + }, + "fsid": "86ebd829-1405-43d3-8fd6-4cbc9b6ecf96", + "keyring": "[osd.3]\n\tkey = AQBBJ/dZp57NIBAAtnuQS9WOS0hnLVe0rZnE6Q==\n", + "kv_backend": "rocksdb", + "magic": "ceph osd volume v026", + "mkfs_done": "yes", + "ready": "ready", + "systemd": "", + "type": "bluestore", + "whoami": "3" + } diff --git a/ceph/doc/ceph-volume/simple/systemd.rst b/ceph/doc/ceph-volume/simple/systemd.rst new file mode 100644 index 000000000..aa5bebffe --- /dev/null +++ b/ceph/doc/ceph-volume/simple/systemd.rst @@ -0,0 +1,28 @@ +.. _ceph-volume-simple-systemd: + +systemd +======= +Upon startup, it will identify the logical volume by loading the JSON file in +``/etc/ceph/osd/{id}-{uuid}.json`` corresponding to the instance name of the +systemd unit. + +After identifying the correct volume it will then proceed to mount it by using +the OSD destination conventions, that is:: + + /var/lib/ceph/osd/{cluster name}-{osd id} + +For our example OSD with an id of ``0``, that means the identified device will +be mounted at:: + + + /var/lib/ceph/osd/ceph-0 + + +Once that process is complete, a call will be made to start the OSD:: + + systemctl start ceph-osd@0 + +The systemd portion of this process is handled by the ``ceph-volume simple +trigger`` sub-command, which is only in charge of parsing metadata coming from +systemd and startup, and then dispatching to ``ceph-volume simple activate`` which +would proceed with activation. diff --git a/ceph/doc/ceph-volume/systemd.rst b/ceph/doc/ceph-volume/systemd.rst new file mode 100644 index 000000000..6cbc11218 --- /dev/null +++ b/ceph/doc/ceph-volume/systemd.rst @@ -0,0 +1,49 @@ +.. _ceph-volume-systemd: + +systemd +======= +As part of the activation process (either with :ref:`ceph-volume-lvm-activate` +or :ref:`ceph-volume-simple-activate`), systemd units will get enabled that +will use the OSD id and uuid as part of their name. These units will be run +when the system boots, and will proceed to activate their corresponding +volumes via their sub-command implementation. + +The API for activation is a bit loose, it only requires two parts: the +subcommand to use and any extra meta information separated by a dash. This +convention makes the units look like:: + + ceph-volume@{command}-{extra metadata} + +The *extra metadata* can be anything needed that the subcommand implementing +the processing might need. In the case of :ref:`ceph-volume-lvm` and +:ref:`ceph-volume-simple`, both look to consume the :term:`OSD id` and :term:`OSD uuid`, +but this is not a hard requirement, it is just how the sub-commands are +implemented. + +Both the command and extra metadata gets persisted by systemd as part of the +*"instance name"* of the unit. For example an OSD with an ID of 0, for the +``lvm`` sub-command would look like:: + + systemctl enable ceph-volume@lvm-0-0A3E1ED2-DA8A-4F0E-AA95-61DEC71768D6 + +The enabled unit is a :term:`systemd oneshot` service, meant to start at boot +after the local filesystem is ready to be used. + + +Failure and Retries +------------------- +It is common to have failures when a system is coming up online. The devices +are sometimes not fully available and this unpredictable behavior may cause an +OSD to not be ready to be used. + +There are two configurable environment variables used to set the retry +behavior: + +* ``CEPH_VOLUME_SYSTEMD_TRIES``: Defaults to 30 +* ``CEPH_VOLUME_SYSTEMD_INTERVAL``: Defaults to 5 + +The *"tries"* is a number that sets the maximum amount of times the unit will +attempt to activate an OSD before giving up. + +The *"interval"* is a value in seconds that determines the waiting time before +initiating another try at activating the OSD. diff --git a/ceph/doc/cephfs/mds-config-ref.rst b/ceph/doc/cephfs/mds-config-ref.rst index 4f7bea3ef..2fd47ae33 100644 --- a/ceph/doc/cephfs/mds-config-ref.rst +++ b/ceph/doc/cephfs/mds-config-ref.rst @@ -613,3 +613,17 @@ :Type: Boolean :Default: ``false`` + + +``mds min caps per client`` + +:Description: Set the minimum number of capabilities a client may hold. +:Type: Integer +:Default: ``100`` + + +``mds max ratio caps per client`` + +:Description: Set the maximum ratio of current caps that may be recalled during MDS cache pressure. +:Type: Float +:Default: ``0.8`` diff --git a/ceph/doc/conf.py b/ceph/doc/conf.py index 49b6ecde2..a1968bb4c 100644 --- a/ceph/doc/conf.py +++ b/ceph/doc/conf.py @@ -41,9 +41,10 @@ extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.graphviz', 'sphinx.ext.todo', - 'sphinx_ditaa', + 'sphinxcontrib.ditaa', 'breathe', ] +ditaa = 'ditaa' todo_include_todos = True top_level = os.path.dirname( diff --git a/ceph/doc/man/8/CMakeLists.txt b/ceph/doc/man/8/CMakeLists.txt index fd6bbae58..f819dede2 100644 --- a/ceph/doc/man/8/CMakeLists.txt +++ b/ceph/doc/man/8/CMakeLists.txt @@ -26,7 +26,8 @@ set(osd_srcs ceph-volume.rst ceph-volume-systemd.rst ceph-osd.rst - osdmaptool.rst) + osdmaptool.rst + ceph-bluestore-tool.rst) set(mon_srcs ceph-mon.rst diff --git a/ceph/doc/man/8/ceph-bluestore-tool.rst b/ceph/doc/man/8/ceph-bluestore-tool.rst new file mode 100644 index 000000000..7a7b0ea6a --- /dev/null +++ b/ceph/doc/man/8/ceph-bluestore-tool.rst @@ -0,0 +1,123 @@ +:orphan: + +====================================================== + ceph-bluestore-tool -- bluestore administrative tool +====================================================== + +.. program:: ceph-bluestore-tool + +Synopsis +======== + +| **ceph-bluestore-tool** *command* + [ --dev *device* ... ] + [ --path *osd path* ] + [ --out-dir *dir* ] + [ --log-file | -l *filename* ] + [ --deep ] +| **ceph-bluestore-tool** fsck|repair --path *osd path* [ --deep ] +| **ceph-bluestore-tool** show-label --dev *device* ... +| **ceph-bluestore-tool** prime-osd-dir --dev *device* --path *osd path* +| **ceph-bluestore-tool** bluefs-export --path *osd path* --out-dir *dir* +| **ceph-bluestore-tool** bluefs-export --path *osd path* --out-dir *dir* + + +Description +=========== + +**ceph-bluestore-tool** is a utility to perform low-level administrative +operations on a BlueStore instance. + +Commands +======== + +.. option:: help + + show help + +.. option:: fsck + + run consistency check on BlueStore metadata. If *--deep* is specified, also read all object data and verify checksums. + +.. option:: repair + + Run a consistency check *and* repair any errors we can. + +.. option:: bluefs-export + + Export the contents of BlueFS (i.e., rocksdb files) to an output directory. + +.. option:: bluefs-bdev-sizes --path *osd path* + + Print the device sizes, as understood by BlueFS, to stdout. + +.. option:: bluefs-bdev-expand --path *osd path* + + Instruct BlueFS to check the size of its block devices and, if they have expanded, make use of the additional space. + +.. option:: show-label --dev *device* [...] + + Show device label(s). + +Options +======= + +.. option:: --dev *device* + + Add *device* to the list of devices to consider + +.. option:: --path *osd path* + + Specify an osd path. In most cases, the device list is inferred from the symlinks present in *osd path*. This is usually simpler than explicitly specifying the device(s) with --dev. + +.. option:: --out-dir *dir* + + Output directory for bluefs-export + +.. option:: -l, --log-file *log file* + + file to log to + +.. option:: --log-level *num* + + debug log level. Default is 30 (extremely verbose), 20 is very + verbose, 10 is verbose, and 1 is not very verbose. + +.. option:: --deep + + deep scrub/repair (read and validate object data, not just metadata) + +Device labels +============= + +Every BlueStore block device has a single block label at the beginning of the +device. You can dump the contents of the label with:: + + ceph-bluestore-tool show-label --dev *device* + +The main device will have a lot of metadata, including information +that used to be stored in small files in the OSD data directory. The +auxilliary devices (db and wal) will only have the minimum required +fields (OSD UUID, size, device type, birth time). + +OSD directory priming +===================== + +You can generate the content for an OSD data directory that can start up a +BlueStore OSD with the *prime-osd-dir* command:: + + ceph-bluestore-tool prime-osd-dir --dev *main device* --path /var/lib/ceph/osd/ceph-*id* + + +Availability +============ + +**ceph-bluestore-tool** is part of Ceph, a massively scalable, +open-source, distributed storage system. Please refer to the Ceph +documentation at http://ceph.com/docs for more information. + + +See also +======== + +:doc:`ceph-osd `\(8) diff --git a/ceph/doc/mgr/administrator.rst b/ceph/doc/mgr/administrator.rst index 1e8d189da..11daf3e42 100644 --- a/ceph/doc/mgr/administrator.rst +++ b/ceph/doc/mgr/administrator.rst @@ -53,6 +53,62 @@ by a standby. If you want to pre-empt failover, you can explicitly mark a ceph-mgr daemon as failed using ``ceph mgr fail ``. +Using modules +------------- + +Use the command ``ceph mgr module ls`` to see which modules are +available, and which are currently enabled. Enable or disable modules +using the commands ``ceph mgr module enable `` and +``ceph mgr module disable `` respectively. + +If a module is *enabled* then the active ceph-mgr daemon will load +and execute it. In the case of modules that provide a service, +such as an HTTP server, the module may publish its address when it +is loaded. To see the addresses of such modules, use the command +``ceph mgr services``. + +Some modules may also implement a special standby mode which runs on +standby ceph-mgr daemons as well as the active daemon. This enables +modules that provide services to redirect their clients to the active +daemon, if the client tries to connect to a standby. + +Consult the documentation pages for individual manager modules for more +information about what functionality each module provides. + +Here is an example of enabling the ``dashboard`` module: + +:: + + $ ceph mgr module ls + { + "enabled_modules": [ + "restful", + "status" + ], + "disabled_modules": [ + "dashboard" + ] + } + + $ ceph mgr module enable dashboard + $ ceph mgr module ls + { + "enabled_modules": [ + "restful", + "status", + "dashboard" + ], + "disabled_modules": [ + ] + } + + $ ceph mgr services + { + "dashboard": "http://myserver.com:7789/", + "restful": "https://myserver.com:8789/" + } + + Calling module commands ----------------------- diff --git a/ceph/doc/mgr/dashboard.rst b/ceph/doc/mgr/dashboard.rst index 890849700..4c2116b13 100644 --- a/ceph/doc/mgr/dashboard.rst +++ b/ceph/doc/mgr/dashboard.rst @@ -39,6 +39,13 @@ If the port is not configured, the web app will bind to port ``7000``. If the address it not configured, the web app will bind to ``::``, which corresponds to all available IPv4 and IPv6 addresses. +You can configure a prefix for all URLs:: + + ceph config-key set mgr/dashboard/url_prefix $PREFIX + +so you can access the dashboard at ``http://$IP:$PORT/$PREFIX/``. + + Load balancer ------------- @@ -48,4 +55,5 @@ manager is active (e.g., ``ceph mgr dump``). In order to make the dashboard available via a consistent URL regardless of which manager daemon is currently active, you may want to set up a load balancer front-end to direct traffic to whichever manager endpoint is -available. +available. If you use a reverse http proxy that forwards a subpath to +the dashboard, you need to configure ``url_prefix`` (see above). diff --git a/ceph/doc/mgr/index.rst b/ceph/doc/mgr/index.rst index 29a221661..53844ba24 100644 --- a/ceph/doc/mgr/index.rst +++ b/ceph/doc/mgr/index.rst @@ -26,9 +26,11 @@ sensible. :maxdepth: 1 Installation and Configuration + Writing plugins Dashboard plugin + Local pool plugin RESTful plugin Zabbix plugin Prometheus plugin - Writing plugins + Influx plugin diff --git a/ceph/doc/mgr/influx.rst b/ceph/doc/mgr/influx.rst new file mode 100644 index 000000000..37aa5cd63 --- /dev/null +++ b/ceph/doc/mgr/influx.rst @@ -0,0 +1,162 @@ +============= +Influx Plugin +============= + +The influx plugin continuously collects and sends time series data to an +influxdb database. + +The influx plugin was introduced in the 13.x *Mimic* release. + +-------- +Enabling +-------- + +To enable the module, use the following command: + +:: + + ceph mgr module enable influx + +If you wish to subsequently disable the module, you can use the equivalent +*disable* command: + +:: + + ceph mgr module disable influx + +------------- +Configuration +------------- + +For the influx module to send statistics to an InfluxDB server, it +is necessary to configure the servers address and some authentication +credentials. + +Set configuration values using the following command: + +:: + + ceph config-key set mgr/influx/ + + +The most important settings are ``hostname``, ``username`` and ``password``. +For example, a typical configuration might look like this: + +:: + + ceph config-key set mgr/influx/hostname influx.mydomain.com + ceph config-key set mgr/influx/username admin123 + ceph config-key set mgr/influx/password p4ssw0rd + +Additional optional configuration settings are: + +:interval: Time between reports to InfluxDB. Default 5 seconds. +:database: InfluxDB database name. Default "ceph". You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it. +:port: InfluxDB server port. Default 8086 + + +--------- +Debugging +--------- + +By default, a few debugging statments as well as error statements have been set to print in the log files. Users can add more if necessary. +To make use of the debugging option in the module: + +- Add this to the ceph.conf file.:: + + [mgr] + debug_mgr = 20 + +- Use this command ``ceph tell mgr. influx self-test``. +- Check the log files. Users may find it easier to filter the log files using *mgr[influx]*. + +-------------------- +Interesting counters +-------------------- + +The following tables describe a subset of the values output by +this module. + +^^^^^ +Pools +^^^^^ + ++---------------+-----------------------------------------------------+ +|Counter | Description | ++===============+=====================================================+ +|bytes_used | Bytes used in the pool not including copies | ++---------------+-----------------------------------------------------+ +|max_avail | Max available number of bytes in the pool | ++---------------+-----------------------------------------------------+ +|objects | Number of objects in the pool | ++---------------+-----------------------------------------------------+ +|wr_bytes | Number of bytes written in the pool | ++---------------+-----------------------------------------------------+ +|dirty | Number of bytes dirty in the pool | ++---------------+-----------------------------------------------------+ +|rd_bytes | Number of bytes read in the pool | ++---------------+-----------------------------------------------------+ +|raw_bytes_used | Bytes used in pool including copies made | ++---------------+-----------------------------------------------------+ + +^^^^ +OSDs +^^^^ + ++------------+------------------------------------+ +|Counter | Description | ++============+====================================+ +|op_w | Client write operations | ++------------+------------------------------------+ +|op_in_bytes | Client operations total write size | ++------------+------------------------------------+ +|op_r | Client read operations | ++------------+------------------------------------+ +|op_out_bytes| Client operations total read size | ++------------+------------------------------------+ + + ++------------------------+--------------------------------------------------------------------------+ +|Counter | Description | ++========================+==========================================================================+ +|op_wip | Replication operations currently being processed (primary) | ++------------------------+--------------------------------------------------------------------------+ +|op_latency | Latency of client operations (including queue time) | ++------------------------+--------------------------------------------------------------------------+ +|op_process_latency | Latency of client operations (excluding queue time) | ++------------------------+--------------------------------------------------------------------------+ +|op_prepare_latency | Latency of client operations (excluding queue time and wait for finished)| ++------------------------+--------------------------------------------------------------------------+ +|op_r_latency | Latency of read operation (including queue time) | ++------------------------+--------------------------------------------------------------------------+ +|op_r_process_latency | Latency of read operation (excluding queue time) | ++------------------------+--------------------------------------------------------------------------+ +|op_w_in_bytes | Client data written | ++------------------------+--------------------------------------------------------------------------+ +|op_w_latency | Latency of write operation (including queue time) | ++------------------------+--------------------------------------------------------------------------+ +|op_w_process_latency | Latency of write operation (excluding queue time) | ++------------------------+--------------------------------------------------------------------------+ +|op_w_prepare_latency | Latency of write operations (excluding queue time and wait for finished) | ++------------------------+--------------------------------------------------------------------------+ +|op_rw | Client read-modify-write operations | ++------------------------+--------------------------------------------------------------------------+ +|op_rw_in_bytes | Client read-modify-write operations write in | ++------------------------+--------------------------------------------------------------------------+ +|op_rw_out_bytes | Client read-modify-write operations read out | ++------------------------+--------------------------------------------------------------------------+ +|op_rw_latency | Latency of read-modify-write operation (including queue time) | ++------------------------+--------------------------------------------------------------------------+ +|op_rw_process_latency | Latency of read-modify-write operation (excluding queue time) | ++------------------------+--------------------------------------------------------------------------+ +|op_rw_prepare_latency | Latency of read-modify-write operations (excluding queue time | +| | and wait for finished) | ++------------------------+--------------------------------------------------------------------------+ +|op_before_queue_op_lat | Latency of IO before calling queue (before really queue into ShardedOpWq)| +| | op_before_dequeue_op_lat | ++------------------------+--------------------------------------------------------------------------+ +|op_before_dequeue_op_lat| Latency of IO before calling dequeue_op(already dequeued and get PG lock)| ++------------------------+--------------------------------------------------------------------------+ + +Latency counters are measured in microseconds unless otherwise specified in the description. + diff --git a/ceph/doc/mgr/localpool.rst b/ceph/doc/mgr/localpool.rst new file mode 100644 index 000000000..5779b7cf1 --- /dev/null +++ b/ceph/doc/mgr/localpool.rst @@ -0,0 +1,35 @@ +Local pool plugin +================= + +The *localpool* plugin can automatically create RADOS pools that are +localized to a subset of the overall cluster. For example, by default, it will +create a pool for each distinct rack in the cluster. This can be useful for some +deployments that want to distribute some data locally as well as globally across the cluster . + +Enabling +-------- + +The *localpool* module is enabled with:: + + ceph mgr module enable localpool + +Configuring +----------- + +The *localpool* module understands the following options: + +* **subtree** (default: `rack`): which CRUSH subtree type the module + should create a pool for. +* **failure_domain** (default: `host`): what failure domain we should + separate data replicas across. +* **pg_num** (default: `128`): number of PGs to create for each pool +* **num_rep** (default: `3`): number of replicas for each pool. + (Currently, pools are always replicated.) +* **min_size** (default: none): value to set min_size to (unchanged from Ceph's default if this option is not set) +* **prefix** (default: `by-$subtreetype-`): prefix for the pool name. + +These options are set via the config-key interface. For example, to +change the replication level to 2x with only 64 PGs, :: + + ceph config-key set mgr/localpool/num_rep 2 + ceph config-key set mgr/localpool/pg_num 64 diff --git a/ceph/doc/mgr/plugins.rst b/ceph/doc/mgr/plugins.rst index b5a13cc3e..a75c14c84 100644 --- a/ceph/doc/mgr/plugins.rst +++ b/ceph/doc/mgr/plugins.rst @@ -157,6 +157,31 @@ a command completes, the ``notify()`` callback on the MgrModule instance is triggered, with notify_type set to "command", and notify_id set to the tag of the command. +Implementing standby mode +------------------------- + +For some modules, it is useful to run on standby manager daemons as well +as on the active daemon. For example, an HTTP server can usefully +serve HTTP redirect responses from the standby managers so that +the user can point his browser at any of the manager daemons without +having to worry about which one is active. + +Standby manager daemons look for a class called ``StandbyModule`` +in each module. If the class is not found then the module is not +used at all on standby daemons. If the class is found, then +its ``serve`` method is called. Implementations of ``StandbyModule`` +must inherit from ``mgr_module.MgrStandbyModule``. + +The interface of ``MgrStandbyModule`` is much restricted compared to +``MgrModule`` -- none of the Ceph cluster state is available to +the module. ``serve`` and ``shutdown`` methods are used in the same +way as a normal module class. The ``get_active_uri`` method enables +the standby module to discover the address of its active peer in +order to make redirects. See the ``MgrStandbyModule`` definition +in the Ceph source code for the full list of methods. + +For an example of how to use this interface, look at the source code +of the ``dashboard`` module. Logging ------- diff --git a/ceph/doc/mgr/prometheus.rst b/ceph/doc/mgr/prometheus.rst index fc84afee4..5bae6a984 100644 --- a/ceph/doc/mgr/prometheus.rst +++ b/ceph/doc/mgr/prometheus.rst @@ -1,3 +1,4 @@ +================= Prometheus plugin ================= @@ -12,8 +13,8 @@ The HTTP path and query parameters are ignored; all extant counters for all reporting entities are returned in text exposition format. (See the Prometheus `documentation `_.) -Enabling --------- +Enabling prometheus output +========================== The *prometheus* module is enabled with:: @@ -28,19 +29,187 @@ configurable with ``ceph config-key set``, with keys ``mgr/prometheus/server_addr`` and ``mgr/prometheus/server_port``. This port is registered with Prometheus's `registry `_. +Statistic names and labels +========================== + +The names of the stats are exactly as Ceph names them, with +illegal characters ``.``, ``-`` and ``::`` translated to ``_``, +and ``ceph_`` prefixed to all names. + + +All *daemon* statistics have a ``ceph_daemon`` label such as "osd.123" +that identifies the type and ID of the daemon they come from. Some +statistics can come from different types of daemon, so when querying +e.g. an OSD's RocksDB stats, you would probably want to filter +on ceph_daemon starting with "osd" to avoid mixing in the monitor +rocksdb stats. + + +The *cluster* statistics (i.e. those global to the Ceph cluster) +have labels appropriate to what they report on. For example, +metrics relating to pools have a ``pool_id`` label. + +Pool and OSD metadata series +---------------------------- + +Special series are output to enable displaying and querying on +certain metadata fields. + +Pools have a ``ceph_pool_metadata`` field like this: + +:: + + ceph_pool_metadata{pool_id="2",name="cephfs_metadata_a"} 0.0 + +OSDs have a ``ceph_osd_metadata`` field like this: + +:: + + ceph_osd_metadata{cluster_addr="172.21.9.34:6802/19096",device_class="ssd",id="0",public_addr="172.21.9.34:6801/19096",weight="1.0"} 0.0 + + +Correlating drive statistics with node_exporter +----------------------------------------------- + +The prometheus output from Ceph is designed to be used in conjunction +with the generic host monitoring from the Prometheus node_exporter. + +To enable correlation of Ceph OSD statistics with node_exporter's +drive statistics, special series are output like this: + +:: + + ceph_disk_occupation{ceph_daemon="osd.0",device="sdd",instance="myhost",job="ceph"} + +To use this to get disk statistics by OSD ID, use the ``and on`` syntax +in your prometheus query like this: + +:: + + rate(node_disk_bytes_written[30s]) and on (device,instance) ceph_disk_occupation{ceph_daemon="osd.0"} + +See the prometheus documentation for more information about constructing +queries. + +Note that for this mechanism to work, Ceph and node_exporter must agree +about the values of the ``instance`` label. See the following section +for guidance about to to set up Prometheus in a way that sets +``instance`` properly. + +Configuring Prometheus server +============================= + +See the prometheus documentation for full details of how to add +scrape endpoints: the notes +in this section are tips on how to configure Prometheus to capture +the Ceph statistics in the most usefully-labelled form. + +This configuration is necessary because Ceph is reporting metrics +from many hosts and services via a single endpoint, and some +metrics that relate to no physical host (such as pool statistics). + +honor_labels +------------ + +To enable Ceph to output properly-labelled data relating to any host, +use the ``honor_labels`` setting when adding the ceph-mgr endpoints +to your prometheus configuration. + +Without this setting, any ``instance`` labels that Ceph outputs, such +as those in ``ceph_disk_occupation`` series, will be overridden +by Prometheus. + +Ceph instance label +------------------- + +By default, Prometheus applies an ``instance`` label that includes +the hostname and port of the endpoint that the series game from. Because +Ceph clusters have multiple manager daemons, this results in an ``instance`` +label that changes spuriously when the active manager daemon changes. + +Set a custom ``instance`` label in your Prometheus target configuration: +you might wish to set it to the hostname of your first monitor, or something +completely arbitrary like "ceph_cluster". + +node_exporter instance labels +----------------------------- + +Set your ``instance`` labels to match what appears in Ceph's OSD metadata +in the ``hostname`` field. This is generally the short hostname of the node. + +This is only necessary if you want to correlate Ceph stats with host stats, +but you may find it useful to do it in all cases in case you want to do +the correlation in the future. + +Example configuration +--------------------- + +This example shows a single node configuration running ceph-mgr and +node_exporter on a server called ``senta04``. + +This is just an example: there are other ways to configure prometheus +scrape targets and label rewrite rules. + +prometheus.yml +~~~~~~~~~~~~~~ + +:: + + global: + scrape_interval: 15s + evaluation_interval: 15s + + scrape_configs: + - job_name: 'node' + file_sd_configs: + - files: + - node_targets.yml + - job_name: 'ceph' + honor_labels: true + file_sd_configs: + - files: + - ceph_targets.yml + + +ceph_targets.yml +~~~~~~~~~~~~~~~~ + + +:: + + [ + { + "targets": [ "senta04.mydomain.com:9283" ], + "labels": { + "instance": "ceph_cluster" + } + } + ] + + +node_targets.yml +~~~~~~~~~~~~~~~~ + +:: + + [ + { + "targets": [ "senta04.mydomain.com:9100" ], + "labels": { + "instance": "senta04" + } + } + ] + + Notes ------ +===== Counters and gauges are exported; currently histograms and long-running averages are not. It's possible that Ceph's 2-D histograms could be reduced to two separate 1-D histograms, and that long-running averages could be exported as Prometheus' Summary type. -The names of the stats are exactly as Ceph names them, with -illegal characters ``.`` and ``-`` translated to ``_``. There is one -label applied, ``daemon``, and its value is the daemon.id for the -daemon in question (e.g. ``{daemon=mon.hosta}`` or ``{daemon=osd.11}``). - Timestamps, as with many Prometheus exporters, are established by the server's scrape time (Prometheus expects that it is polling the actual counter process synchronously). It is possible to supply a diff --git a/ceph/doc/rados/configuration/pool-pg-config-ref.rst b/ceph/doc/rados/configuration/pool-pg-config-ref.rst index dd416edfa..89a3707cc 100644 --- a/ceph/doc/rados/configuration/pool-pg-config-ref.rst +++ b/ceph/doc/rados/configuration/pool-pg-config-ref.rst @@ -255,6 +255,15 @@ Ceph configuration file. :Type: 32-bit Integer :Default: ``45`` +``osd max pg per osd hard ratio`` + +:Description: The ratio of number of PGs per OSD allowed by the cluster before + OSD refuses to create new PGs. OSD stops creating new PGs if the number + of PGs it serves exceeds + ``osd max pg per osd hard ratio`` \* ``mon max pg per osd``. + +:Type: Float +:Default: ``2`` .. _pool: ../../operations/pools .. _Monitoring OSDs and PGs: ../../operations/monitoring-osd-pg#peering diff --git a/ceph/doc/rados/operations/health-checks.rst b/ceph/doc/rados/operations/health-checks.rst index 616435579..c1e22004a 100644 --- a/ceph/doc/rados/operations/health-checks.rst +++ b/ceph/doc/rados/operations/health-checks.rst @@ -336,17 +336,20 @@ TOO_MANY_PGS ____________ The number of PGs in use in the cluster is above the configurable -threshold of ``mon_pg_warn_max_per_osd`` PGs per OSD. This can lead +threshold of ``mon_max_pg_per_osd`` PGs per OSD. If this threshold is +exceed the cluster will not allow new pools to be created, pool `pg_num` to +be increased, or pool replication to be increased (any of which would lead to +more PGs in the cluster). A large number of PGs can lead to higher memory utilization for OSD daemons, slower peering after cluster state changes (like OSD restarts, additions, or removals), and higher load on the Manager and Monitor daemons. -The ``pg_num`` value for existing pools cannot currently be reduced. -However, the ``pgp_num`` value can, which effectively collocates some -PGs on the same sets of OSDs, mitigating some of the negative impacts -described above. The ``pgp_num`` value can be adjusted with:: +The simplest way to mitigate the problem is to increase the number of +OSDs in the cluster by adding more hardware. Note that the OSD count +used for the purposes of this health check is the number of "in" OSDs, +so marking "out" OSDs "in" (if there are any) can also help:: - ceph osd pool set pgp_num + ceph osd in Please refer to :doc:`placement-groups#Choosing-the-number-of-Placement-Groups` for @@ -368,7 +371,6 @@ triggering the data migration, with:: ceph osd pool set pgp_num - MANY_OBJECTS_PER_PG ___________________ diff --git a/ceph/doc/scripts/gen_state_diagram.py b/ceph/doc/scripts/gen_state_diagram.py index f8414368c..fccde2629 100755 --- a/ceph/doc/scripts/gen_state_diagram.py +++ b/ceph/doc/scripts/gen_state_diagram.py @@ -82,14 +82,22 @@ class StateMachineRenderer(object): ) def read_input(self, input_lines): + previous_line = None for line in input_lines: self.get_state(line) self.get_event(line) - self.get_context(line) - - def get_context(self, line): - match = re.search(r"(\w+::)*::(?P\w+)::\w+\(const (?P\w+)", - line) + # pass two lines at a time to get the context so that regexes can + # match on split signatures + self.get_context(line, previous_line) + previous_line = line + + def get_context(self, line, previous_line): + match = re.search(r"(\w+::)*::(?P\w+)::\w+\(const (?P\w+)", line) + if match is None and previous_line is not None: + # it is possible that we need to match on the previous line as well, so join + # them to make them one line and try and get this matching + joined_line = ' '.join([previous_line, line]) + match = re.search(r"(\w+::)*::(?P\w+)::\w+\(\s*const (?P\w+)", joined_line) if match is not None: self.context.append((match.group('tag'), self.context_depth, match.group('event'))) if '{' in line: @@ -105,7 +113,7 @@ class StateMachineRenderer(object): r"boost::statechart::state_machine<\s*(\w*),\s*(\w*)\s*>", line) if tokens is None: - raise "Error: malformed state_machine line: " + line + raise Exception("Error: malformed state_machine line: " + line) self.machines[tokens.group(1)] = tokens.group(2) self.context.append((tokens.group(1), self.context_depth, "")) return @@ -114,7 +122,7 @@ class StateMachineRenderer(object): r"boost::statechart::state<\s*(\w*),\s*(\w*)\s*,?\s*(\w*)\s*>", line) if tokens is None: - raise "Error: malformed state line: " + line + raise Exception("Error: malformed state line: " + line) self.states[tokens.group(1)] = tokens.group(2) if tokens.group(2) not in self.state_contents.keys(): self.state_contents[tokens.group(2)] = [] @@ -131,14 +139,14 @@ class StateMachineRenderer(object): if i.group(1) not in self.edges.keys(): self.edges[i.group(1)] = [] if len(self.context) is 0: - raise "no context at line: " + line + raise Exception("no context at line: " + line) self.edges[i.group(1)].append((self.context[-1][0], i.group(2))) i = re.search("return\s+transit<\s*(\w*)\s*>()", line) if i is not None: if len(self.context) is 0: - raise "no context at line: " + line + raise Exception("no context at line: " + line) if self.context[-1][2] is "": - raise "no event in context at line: " + line + raise Exception("no event in context at line: " + line) if self.context[-1][2] not in self.edges.keys(): self.edges[self.context[-1][2]] = [] self.edges[self.context[-1][2]].append((self.context[-1][0], i.group(1))) diff --git a/ceph/etc/default/ceph b/ceph/etc/default/ceph index f2722073b..4542838f4 100644 --- a/ceph/etc/default/ceph +++ b/ceph/etc/default/ceph @@ -5,11 +5,3 @@ # Increase tcmalloc cache size TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728 - -## use jemalloc instead of tcmalloc -# -# jemalloc is generally faster for small IO workloads and when -# ceph-osd is backed by SSDs. However, memory usage is usually -# higher by 200-300mb. -# -#LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 diff --git a/ceph/etc/sysconfig/ceph b/ceph/etc/sysconfig/ceph index 61e941ded..c7f4bc45a 100644 --- a/ceph/etc/sysconfig/ceph +++ b/ceph/etc/sysconfig/ceph @@ -6,14 +6,6 @@ # Increase tcmalloc cache size TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728 -## use jemalloc instead of tcmalloc -# -# jemalloc is generally faster for small IO workloads and when -# ceph-osd is backed by SSDs. However, memory usage is usually -# higher by 200-300mb. -# -#LD_PRELOAD=/usr/lib64/libjemalloc.so.1 - ## automatically restart systemd units on upgrade # # By default, it is left to the administrator to restart diff --git a/ceph/qa/cephfs/clusters/3-mds.yaml b/ceph/qa/cephfs/clusters/3-mds.yaml index ff35ed1a6..05c6142f8 100644 --- a/ceph/qa/cephfs/clusters/3-mds.yaml +++ b/ceph/qa/cephfs/clusters/3-mds.yaml @@ -1,4 +1,4 @@ roles: -- [mon.a, mon.c, mgr.y, mds.a, osd.0, osd.1, osd.2] -- [mon.b, mgr.x, mds.b, mds.c, osd.3, osd.4, osd.5] +- [mon.a, mon.c, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3] +- [mon.b, mgr.x, mds.b, mds.c, osd.4, osd.5, osd.6, osd.7] - [client.0] diff --git a/ceph/qa/cephfs/clusters/9-mds.yaml b/ceph/qa/cephfs/clusters/9-mds.yaml index c1228b3a1..a6342dc06 100644 --- a/ceph/qa/cephfs/clusters/9-mds.yaml +++ b/ceph/qa/cephfs/clusters/9-mds.yaml @@ -1,4 +1,4 @@ roles: -- [mon.a, mon.c, mgr.y, mds.a, mds.b, mds.c, mds.d, osd.0, osd.1, osd.2] -- [mon.b, mgr.x, mds.e, mds.f, mds.g, mds.h, mds.i, osd.3, osd.4, osd.5] +- [mon.a, mon.c, mgr.y, mds.a, mds.b, mds.c, mds.d, osd.0, osd.1, osd.2, osd.3] +- [mon.b, mgr.x, mds.e, mds.f, mds.g, mds.h, mds.i, osd.4, osd.5, osd.6, osd.7] - [client.0] diff --git a/ceph/qa/cephfs/objectstore-ec/bluestore-comp-ec-root.yaml b/ceph/qa/cephfs/objectstore-ec/bluestore-comp-ec-root.yaml new file mode 100644 index 000000000..9bc487cfc --- /dev/null +++ b/ceph/qa/cephfs/objectstore-ec/bluestore-comp-ec-root.yaml @@ -0,0 +1,28 @@ +overrides: + thrashosds: + bdev_inject_crash: 2 + bdev_inject_crash_probability: .5 + ceph: + fs: xfs + cephfs_ec_profile: + - m=2 + - k=2 + - crush-failure-domain=osd + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore compression mode: aggressive + bluestore fsck on mount: true + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 + +# this doesn't work with failures bc the log writes are not atomic across the two backends +# bluestore bluefs env mirror: true + diff --git a/ceph/qa/cephfs/objectstore-ec/bluestore-comp.yaml b/ceph/qa/cephfs/objectstore-ec/bluestore-comp.yaml new file mode 100644 index 000000000..b408032fd --- /dev/null +++ b/ceph/qa/cephfs/objectstore-ec/bluestore-comp.yaml @@ -0,0 +1,23 @@ +overrides: + thrashosds: + bdev_inject_crash: 2 + bdev_inject_crash_probability: .5 + ceph: + fs: xfs + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore compression mode: aggressive + bluestore fsck on mount: true + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 + +# this doesn't work with failures bc the log writes are not atomic across the two backends +# bluestore bluefs env mirror: true diff --git a/ceph/qa/cephfs/objectstore-ec/bluestore-ec-root.yaml b/ceph/qa/cephfs/objectstore-ec/bluestore-ec-root.yaml new file mode 100644 index 000000000..726ad3d56 --- /dev/null +++ b/ceph/qa/cephfs/objectstore-ec/bluestore-ec-root.yaml @@ -0,0 +1,42 @@ +overrides: + thrashosds: + bdev_inject_crash: 2 + bdev_inject_crash_probability: .5 + ceph: + fs: xfs + cephfs_ec_profile: + - m=2 + - k=2 + - crush-failure-domain=osd + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore fsck on mount: true + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 +# this doesn't work with failures bc the log writes are not atomic across the two backends +# bluestore bluefs env mirror: true + ceph-deploy: + fs: xfs + bluestore: yes + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore fsck on mount: true + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 + diff --git a/ceph/qa/cephfs/objectstore-ec/bluestore.yaml b/ceph/qa/cephfs/objectstore-ec/bluestore.yaml new file mode 100644 index 000000000..19dfeb036 --- /dev/null +++ b/ceph/qa/cephfs/objectstore-ec/bluestore.yaml @@ -0,0 +1,38 @@ +overrides: + thrashosds: + bdev_inject_crash: 2 + bdev_inject_crash_probability: .5 + ceph: + fs: xfs + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore fsck on mount: true + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 +# this doesn't work with failures bc the log writes are not atomic across the two backends +# bluestore bluefs env mirror: true + ceph-deploy: + fs: xfs + bluestore: yes + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore fsck on mount: true + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 + diff --git a/ceph/qa/cephfs/objectstore-ec/filestore-xfs.yaml b/ceph/qa/cephfs/objectstore-ec/filestore-xfs.yaml new file mode 100644 index 000000000..f7aa0dd79 --- /dev/null +++ b/ceph/qa/cephfs/objectstore-ec/filestore-xfs.yaml @@ -0,0 +1,15 @@ +overrides: + ceph: + fs: xfs + conf: + osd: + osd objectstore: filestore + osd sloppy crc: true + ceph-deploy: + fs: xfs + filestore: True + conf: + osd: + osd objectstore: filestore + osd sloppy crc: true + diff --git a/ceph/qa/distros/all/centos_7.4.yaml b/ceph/qa/distros/all/centos_7.4.yaml new file mode 100644 index 000000000..d06bc384b --- /dev/null +++ b/ceph/qa/distros/all/centos_7.4.yaml @@ -0,0 +1,2 @@ +os_type: centos +os_version: "7.4" diff --git a/ceph/qa/distros/supported/centos_latest.yaml b/ceph/qa/distros/supported/centos_latest.yaml index 34d81be44..4cc59dad1 120000 --- a/ceph/qa/distros/supported/centos_latest.yaml +++ b/ceph/qa/distros/supported/centos_latest.yaml @@ -1 +1 @@ -../all/centos_7.3.yaml \ No newline at end of file +../all/centos_7.4.yaml \ No newline at end of file diff --git a/ceph/qa/releases/luminous-with-mgr.yaml b/ceph/qa/releases/luminous-with-mgr.yaml index ea3130768..391a5e181 100644 --- a/ceph/qa/releases/luminous-with-mgr.yaml +++ b/ceph/qa/releases/luminous-with-mgr.yaml @@ -2,10 +2,11 @@ tasks: - exec: osd.0: - ceph osd require-osd-release luminous - - ceph osd set-require-min-compat-client luminous - ceph.healthy: overrides: ceph: conf: mon: mon warn on osd down out interval zero: false + log-whitelist: + - ruleset- diff --git a/ceph/qa/releases/luminous.yaml b/ceph/qa/releases/luminous.yaml index 9ed76715a..5bd666ca0 100644 --- a/ceph/qa/releases/luminous.yaml +++ b/ceph/qa/releases/luminous.yaml @@ -19,3 +19,4 @@ overrides: mon warn on osd down out interval zero: false log-whitelist: - no active mgr + - ruleset- diff --git a/ceph/qa/standalone/mon/osd-pool-create.sh b/ceph/qa/standalone/mon/osd-pool-create.sh index 99a6064c0..693165d89 100755 --- a/ceph/qa/standalone/mon/osd-pool-create.sh +++ b/ceph/qa/standalone/mon/osd-pool-create.sh @@ -200,7 +200,7 @@ function TEST_utf8_cli() { # the fix for http://tracker.ceph.com/issues/7387. If it turns out # to not be OK (when is the default encoding *not* UTF-8?), maybe # the character '黄' can be replaced with the escape $'\xe9\xbb\x84' - ceph osd pool create 黄 1024 || return 1 + ceph osd pool create 黄 16 || return 1 ceph osd lspools 2>&1 | \ grep "黄" || return 1 ceph -f json-pretty osd dump | \ diff --git a/ceph/src/test/ceph_objectstore_tool.py b/ceph/qa/standalone/special/ceph_objectstore_tool.py similarity index 97% rename from ceph/src/test/ceph_objectstore_tool.py rename to ceph/qa/standalone/special/ceph_objectstore_tool.py index bae12f4d8..7c52101e4 100755 --- a/ceph/src/test/ceph_objectstore_tool.py +++ b/ceph/qa/standalone/special/ceph_objectstore_tool.py @@ -152,7 +152,7 @@ def cat_file(level, filename): def vstart(new, opt=""): print("vstarting....", end="") NEW = new and "-n" or "-N" - call("MON=1 OSD=4 MDS=0 MGR=1 CEPH_PORT=7400 {path}/src/vstart.sh --short -l {new} -d {opt} > /dev/null 2>&1".format(new=NEW, opt=opt, path=CEPH_ROOT), shell=True) + call("MON=1 OSD=4 MDS=0 MGR=1 CEPH_PORT=7400 {path}/src/vstart.sh --filestore --short -l {new} -d {opt} > /dev/null 2>&1".format(new=NEW, opt=opt, path=CEPH_ROOT), shell=True) print("DONE") @@ -388,14 +388,18 @@ CEPH_ROOT = os.environ.get('CEPH_ROOT') if not CEPH_BUILD_DIR: CEPH_BUILD_DIR=os.getcwd() os.putenv('CEPH_BUILD_DIR', CEPH_BUILD_DIR) - CEPH_BIN=CEPH_BUILD_DIR + CEPH_BIN=os.path.join(CEPH_BUILD_DIR, 'bin') os.putenv('CEPH_BIN', CEPH_BIN) CEPH_ROOT=os.path.dirname(CEPH_BUILD_DIR) os.putenv('CEPH_ROOT', CEPH_ROOT) - CEPH_LIB=os.path.join(CEPH_BIN, '.libs') + CEPH_LIB=os.path.join(CEPH_BUILD_DIR, 'lib') os.putenv('CEPH_LIB', CEPH_LIB) -CEPH_DIR = CEPH_BUILD_DIR + "/cot_dir" +try: + os.mkdir("td") +except: + pass # ok if this is already there +CEPH_DIR = os.path.join(CEPH_BUILD_DIR, os.path.join("td", "cot_dir")) CEPH_CONF = os.path.join(CEPH_DIR, 'ceph.conf') def kill_daemons(): @@ -518,7 +522,7 @@ def get_osd_weights(CFSD_PREFIX, osd_ids, osd_path): if linev[0] is '': linev.pop(0) print('linev %s' % linev) - weights.append(float(linev[1])) + weights.append(float(linev[2])) return weights @@ -672,9 +676,10 @@ def main(argv): else: nullfd = DEVNULL - call("rm -fr {dir}; mkdir {dir}".format(dir=CEPH_DIR), shell=True) + call("rm -fr {dir}; mkdir -p {dir}".format(dir=CEPH_DIR), shell=True) + os.chdir(CEPH_DIR) os.environ["CEPH_DIR"] = CEPH_DIR - OSDDIR = os.path.join(CEPH_DIR, "dev") + OSDDIR = "dev" REP_POOL = "rep_pool" REP_NAME = "REPobject" EC_POOL = "ec_pool" @@ -713,6 +718,7 @@ def main(argv): cmd = "{path}/ceph osd pool create {pool} {pg} {pg} replicated".format(pool=REP_POOL, pg=PG_COUNT, path=CEPH_BIN) logging.debug(cmd) call(cmd, shell=True, stdout=nullfd, stderr=nullfd) + time.sleep(2) REPID = get_pool_id(REP_POOL, nullfd) print("Created Replicated pool #{repid}".format(repid=REPID)) @@ -989,6 +995,12 @@ def main(argv): cmd = "{path}/ceph-objectstore-tool --journal-path BAD_JOURNAL_PATH --op dump-journal".format(path=CEPH_BIN) ERRORS += test_failure(cmd, "journal-path: BAD_JOURNAL_PATH: (2) No such file or directory") + cmd = (CFSD_PREFIX + "--journal-path BAD_JOURNAL_PATH --op list").format(osd=ONEOSD) + ERRORS += test_failure(cmd, "journal-path: BAD_JOURNAL_PATH: No such file or directory") + + cmd = (CFSD_PREFIX + "--journal-path /bin --op list").format(osd=ONEOSD) + ERRORS += test_failure(cmd, "journal-path: /bin: (21) Is a directory") + # On import can't use stdin from a terminal cmd = (CFSD_PREFIX + "--op import --pgid {pg}").format(osd=ONEOSD, pg=ONEPG) ERRORS += test_failure(cmd, "stdin is a tty and no --file filename specified", tty=True) @@ -1006,7 +1018,10 @@ def main(argv): cmd = "{path}/ceph-objectstore-tool --type memstore --op list --pgid {pg}".format(dir=OSDDIR, osd=ONEOSD, pg=ONEPG, path=CEPH_BIN) ERRORS += test_failure(cmd, "Must provide --data-path") - cmd = (CFSD_PREFIX + "--op remove").format(osd=ONEOSD) + cmd = (CFSD_PREFIX + "--op remove --pgid 2.0").format(osd=ONEOSD) + ERRORS += test_failure(cmd, "Please use export-remove or you must use --force option") + + cmd = (CFSD_PREFIX + "--force --op remove").format(osd=ONEOSD) ERRORS += test_failure(cmd, "Must provide pgid") # Don't secify a --op nor object command @@ -1015,7 +1030,7 @@ def main(argv): # Specify a bad --op command cmd = (CFSD_PREFIX + "--op oops").format(osd=ONEOSD) - ERRORS += test_failure(cmd, "Must provide --op (info, log, remove, mkfs, fsck, export, import, list, fix-lost, list-pgs, rm-past-intervals, dump-journal, dump-super, meta-list, get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete)") + ERRORS += test_failure(cmd, "Must provide --op (info, log, remove, mkfs, fsck, export, export-remove, import, list, fix-lost, list-pgs, rm-past-intervals, dump-journal, dump-super, meta-list, get-osdmap, set-osdmap, get-inc-osdmap, set-inc-osdmap, mark-complete)") # Provide just the object param not a command cmd = (CFSD_PREFIX + "object").format(osd=ONEOSD) @@ -1720,7 +1735,7 @@ def main(argv): if ret != 0: logging.error("Removing --dry-run failed for pg {pg} on {osd} with {ret}".format(pg=pg, osd=osd, ret=ret)) RM_ERRORS += 1 - cmd = (CFSD_PREFIX + "--op remove --pgid {pg}").format(pg=pg, osd=osd) + cmd = (CFSD_PREFIX + "--force --op remove --pgid {pg}").format(pg=pg, osd=osd) logging.debug(cmd) ret = call(cmd, shell=True, stdout=nullfd) if ret != 0: @@ -1923,7 +1938,7 @@ def main(argv): which = 0 for osd in get_osds(pg, OSDDIR): - cmd = (CFSD_PREFIX + "--op remove --pgid {pg}").format(pg=pg, osd=osd) + cmd = (CFSD_PREFIX + "--force --op remove --pgid {pg}").format(pg=pg, osd=osd) logging.debug(cmd) ret = call(cmd, shell=True, stdout=nullfd) @@ -1961,6 +1976,17 @@ def main(argv): # vstart() starts 4 OSDs ERRORS += test_get_set_osdmap(CFSD_PREFIX, list(range(4)), ALLOSDS) ERRORS += test_get_set_inc_osdmap(CFSD_PREFIX, ALLOSDS[0]) + + kill_daemons() + CORES = [f for f in os.listdir(CEPH_DIR) if f.startswith("core.")] + if CORES: + CORE_DIR = os.path.join("/tmp", "cores.{pid}".format(pid=os.getpid())) + os.mkdir(CORE_DIR) + call("/bin/mv {ceph_dir}/core.* {core_dir}".format(ceph_dir=CEPH_DIR, core_dir=CORE_DIR), shell=True) + logging.error("Failure due to cores found") + logging.error("See {core_dir} for cores".format(core_dir=CORE_DIR)) + ERRORS += len(CORES) + if ERRORS == 0: print("TEST PASSED") return 0 @@ -1992,6 +2018,7 @@ if __name__ == "__main__": status = main(sys.argv[1:]) finally: kill_daemons() + os.chdir(CEPH_BUILD_DIR) remove_btrfs_subvolumes(CEPH_DIR) call("/bin/rm -fr {dir}".format(dir=CEPH_DIR), shell=True) sys.exit(status) diff --git a/ceph/qa/suites/ceph-ansible/smoke/basic/1-distros/centos_7.3.yaml b/ceph/qa/suites/ceph-ansible/smoke/basic/1-distros/centos_7.3.yaml deleted file mode 100644 index 9dfcc7f6b..000000000 --- a/ceph/qa/suites/ceph-ansible/smoke/basic/1-distros/centos_7.3.yaml +++ /dev/null @@ -1,2 +0,0 @@ -os_type: centos -os_version: "7.3" diff --git a/ceph/qa/suites/ceph-ansible/smoke/basic/1-distros/centos_latest.yaml b/ceph/qa/suites/ceph-ansible/smoke/basic/1-distros/centos_latest.yaml new file mode 120000 index 000000000..b5973b952 --- /dev/null +++ b/ceph/qa/suites/ceph-ansible/smoke/basic/1-distros/centos_latest.yaml @@ -0,0 +1 @@ +../../../../../distros/supported/centos_latest.yaml \ No newline at end of file diff --git a/ceph/qa/suites/ceph-ansible/smoke/basic/1-distros/ubuntu_16.04.yaml b/ceph/qa/suites/ceph-ansible/smoke/basic/1-distros/ubuntu_16.04.yaml deleted file mode 100644 index a459fddff..000000000 --- a/ceph/qa/suites/ceph-ansible/smoke/basic/1-distros/ubuntu_16.04.yaml +++ /dev/null @@ -1,2 +0,0 @@ -os_type: ubuntu -os_version: "16.04" diff --git a/ceph/qa/suites/ceph-ansible/smoke/basic/1-distros/ubuntu_latest.yaml b/ceph/qa/suites/ceph-ansible/smoke/basic/1-distros/ubuntu_latest.yaml new file mode 120000 index 000000000..cc5b15bcc --- /dev/null +++ b/ceph/qa/suites/ceph-ansible/smoke/basic/1-distros/ubuntu_latest.yaml @@ -0,0 +1 @@ +../../../../../distros/supported/ubuntu_latest.yaml \ No newline at end of file diff --git a/ceph/qa/suites/ceph-ansible/smoke/basic/2-ceph/ceph_ansible.yaml b/ceph/qa/suites/ceph-ansible/smoke/basic/2-ceph/ceph_ansible.yaml new file mode 100644 index 000000000..36d0a07d9 --- /dev/null +++ b/ceph/qa/suites/ceph-ansible/smoke/basic/2-ceph/ceph_ansible.yaml @@ -0,0 +1,32 @@ +meta: +- desc: "Build the ceph cluster using ceph-ansible" + +overrides: + ceph_ansible: + vars: + ceph_conf_overrides: + global: + osd default pool size: 2 + mon pg warn min per osd: 2 + osd pool default pg num: 64 + osd pool default pgp num: 64 + mon_max_pg_per_osd: 1024 + ceph_test: true + ceph_stable_release: luminous + osd_scenario: collocated + journal_size: 1024 + osd_auto_discovery: false + ceph_origin: repository + ceph_repository: dev + ceph_mgr_modules: + - status + - restful + cephfs_pools: + - name: "cephfs_data" + pgs: "64" + - name: "cephfs_metadata" + pgs: "64" +tasks: +- ssh-keys: +- ceph_ansible: +- install.ship_utilities: diff --git a/ceph/qa/suites/ceph-ansible/smoke/basic/2-config/ceph_ansible.yaml b/ceph/qa/suites/ceph-ansible/smoke/basic/2-config/ceph_ansible.yaml deleted file mode 100644 index 5750d52bc..000000000 --- a/ceph/qa/suites/ceph-ansible/smoke/basic/2-config/ceph_ansible.yaml +++ /dev/null @@ -1,22 +0,0 @@ -meta: -- desc: "Build the ceph cluster using ceph-ansible" - -overrides: - ceph_ansible: - vars: - ceph_conf_overrides: - global: - osd default pool size: 2 - mon pg warn min per osd: 2 - ceph_dev: true - ceph_dev_key: https://download.ceph.com/keys/autobuild.asc - ceph_origin: upstream - ceph_test: true - journal_collocation: true - journal_size: 1024 - osd_auto_discovery: false - -tasks: -- ssh-keys: -- ceph_ansible: -- install.ship_utilities: diff --git a/ceph/qa/suites/ceph-ansible/smoke/basic/3-config/bluestore_with_dmcrypt.yaml b/ceph/qa/suites/ceph-ansible/smoke/basic/3-config/bluestore_with_dmcrypt.yaml new file mode 100644 index 000000000..604e757ad --- /dev/null +++ b/ceph/qa/suites/ceph-ansible/smoke/basic/3-config/bluestore_with_dmcrypt.yaml @@ -0,0 +1,8 @@ +meta: +- desc: "use bluestore + dmcrypt option" + +overrides: + ceph_ansible: + vars: + osd_objectstore: bluestore + dmcrypt: True diff --git a/ceph/qa/suites/ceph-ansible/smoke/basic/3-config/dmcrypt_off.yaml b/ceph/qa/suites/ceph-ansible/smoke/basic/3-config/dmcrypt_off.yaml new file mode 100644 index 000000000..4bbd1c7c5 --- /dev/null +++ b/ceph/qa/suites/ceph-ansible/smoke/basic/3-config/dmcrypt_off.yaml @@ -0,0 +1,7 @@ +meta: +- desc: "without dmcrypt" + +overrides: + ceph_ansible: + vars: + dmcrypt: False diff --git a/ceph/qa/suites/ceph-ansible/smoke/basic/3-config/dmcrypt_on.yaml b/ceph/qa/suites/ceph-ansible/smoke/basic/3-config/dmcrypt_on.yaml new file mode 100644 index 000000000..12d63d325 --- /dev/null +++ b/ceph/qa/suites/ceph-ansible/smoke/basic/3-config/dmcrypt_on.yaml @@ -0,0 +1,7 @@ +meta: +- desc: "use dmcrypt option" + +overrides: + ceph_ansible: + vars: + dmcrypt: True diff --git a/ceph/qa/suites/ceph-ansible/smoke/basic/3-tasks/cls.yaml b/ceph/qa/suites/ceph-ansible/smoke/basic/3-tasks/cls.yaml deleted file mode 100644 index 781a4d4f8..000000000 --- a/ceph/qa/suites/ceph-ansible/smoke/basic/3-tasks/cls.yaml +++ /dev/null @@ -1,7 +0,0 @@ -meta: -- desc: "Run the rados cls tests" -tasks: -- workunit: - clients: - client.0: - - cls diff --git a/ceph/qa/suites/ceph-ansible/smoke/basic/3-tasks/ceph-admin-commands.yaml b/ceph/qa/suites/ceph-ansible/smoke/basic/4-tasks/ceph-admin-commands.yaml similarity index 100% rename from ceph/qa/suites/ceph-ansible/smoke/basic/3-tasks/ceph-admin-commands.yaml rename to ceph/qa/suites/ceph-ansible/smoke/basic/4-tasks/ceph-admin-commands.yaml diff --git a/ceph/qa/suites/ceph-ansible/smoke/basic/3-tasks/rbd_import_export.yaml b/ceph/qa/suites/ceph-ansible/smoke/basic/4-tasks/rbd_import_export.yaml similarity index 100% rename from ceph/qa/suites/ceph-ansible/smoke/basic/3-tasks/rbd_import_export.yaml rename to ceph/qa/suites/ceph-ansible/smoke/basic/4-tasks/rbd_import_export.yaml diff --git a/ceph/qa/suites/ceph-ansible/smoke/basic/4-tasks/rest.yaml b/ceph/qa/suites/ceph-ansible/smoke/basic/4-tasks/rest.yaml new file mode 100644 index 000000000..8e389134b --- /dev/null +++ b/ceph/qa/suites/ceph-ansible/smoke/basic/4-tasks/rest.yaml @@ -0,0 +1,15 @@ +tasks: +- exec: + mgr.x: + - systemctl stop ceph-mgr.target + - sleep 5 + - ceph -s +- exec: + mon.a: + - ceph restful create-key admin + - ceph restful create-self-signed-cert + - ceph restful restart +- workunit: + clients: + client.0: + - rest/test-restful.sh diff --git a/ceph/qa/suites/fs/32bits/objectstore b/ceph/qa/suites/fs/32bits/objectstore deleted file mode 120000 index c72da2f82..000000000 --- a/ceph/qa/suites/fs/32bits/objectstore +++ /dev/null @@ -1 +0,0 @@ -../../../objectstore_cephfs \ No newline at end of file diff --git a/ceph/qa/suites/fs/32bits/objectstore-ec b/ceph/qa/suites/fs/32bits/objectstore-ec new file mode 120000 index 000000000..15dc98f23 --- /dev/null +++ b/ceph/qa/suites/fs/32bits/objectstore-ec @@ -0,0 +1 @@ +../../../cephfs/objectstore-ec \ No newline at end of file diff --git a/ceph/qa/suites/fs/basic_functional/clusters/4-remote-clients.yaml b/ceph/qa/suites/fs/basic_functional/clusters/4-remote-clients.yaml index f9e423e2f..1c540a4ef 100644 --- a/ceph/qa/suites/fs/basic_functional/clusters/4-remote-clients.yaml +++ b/ceph/qa/suites/fs/basic_functional/clusters/4-remote-clients.yaml @@ -1,6 +1,6 @@ roles: -- [mon.a, mgr.x, osd.0, mds.a, mds.b, client.1, client.2, client.3] -- [client.0, osd.1, osd.2] +- [mon.a, mgr.x, osd.0, osd.1, osd.2, osd.3, mds.a, mds.b, client.1, client.2, client.3] +- [client.0, osd.4, osd.5, osd.6, osd.7] openstack: - volumes: # attached to each instance count: 2 diff --git a/ceph/qa/suites/fs/basic_functional/objectstore/bluestore-ec-root.yaml b/ceph/qa/suites/fs/basic_functional/objectstore/bluestore-ec-root.yaml new file mode 120000 index 000000000..36a4d69cd --- /dev/null +++ b/ceph/qa/suites/fs/basic_functional/objectstore/bluestore-ec-root.yaml @@ -0,0 +1 @@ +../../../../cephfs/objectstore-ec/bluestore-ec-root.yaml \ No newline at end of file diff --git a/ceph/qa/suites/fs/basic_workload/objectstore b/ceph/qa/suites/fs/basic_workload/objectstore deleted file mode 120000 index c72da2f82..000000000 --- a/ceph/qa/suites/fs/basic_workload/objectstore +++ /dev/null @@ -1 +0,0 @@ -../../../objectstore_cephfs \ No newline at end of file diff --git a/ceph/qa/suites/fs/basic_workload/objectstore-ec b/ceph/qa/suites/fs/basic_workload/objectstore-ec new file mode 120000 index 000000000..a330d661a --- /dev/null +++ b/ceph/qa/suites/fs/basic_workload/objectstore-ec @@ -0,0 +1 @@ +../../../cephfs/objectstore-ec/ \ No newline at end of file diff --git a/ceph/qa/suites/fs/multiclient/clusters/three_clients.yaml b/ceph/qa/suites/fs/multiclient/clusters/three_clients.yaml index 78b912f72..a533af5c6 100644 --- a/ceph/qa/suites/fs/multiclient/clusters/three_clients.yaml +++ b/ceph/qa/suites/fs/multiclient/clusters/three_clients.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, mon.b, mon.c, mgr.x, mds.a, osd.0, osd.1, osd.2] +- [mon.a, mon.b, mon.c, mgr.x, mds.a, osd.0, osd.1, osd.2, osd.3] - [client.2] - [client.1] - [client.0] diff --git a/ceph/qa/suites/fs/multiclient/clusters/two_clients.yaml b/ceph/qa/suites/fs/multiclient/clusters/two_clients.yaml index 9586e6c8f..00f3815cb 100644 --- a/ceph/qa/suites/fs/multiclient/clusters/two_clients.yaml +++ b/ceph/qa/suites/fs/multiclient/clusters/two_clients.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, mon.b, mon.c, mgr.x, mds.a, osd.0, osd.1, osd.2] +- [mon.a, mon.b, mon.c, mgr.x, mds.a, osd.0, osd.1, osd.2, osd.3] - [client.1] - [client.0] diff --git a/ceph/qa/suites/fs/multiclient/objectstore b/ceph/qa/suites/fs/multiclient/objectstore deleted file mode 120000 index c72da2f82..000000000 --- a/ceph/qa/suites/fs/multiclient/objectstore +++ /dev/null @@ -1 +0,0 @@ -../../../objectstore_cephfs \ No newline at end of file diff --git a/ceph/qa/suites/fs/multiclient/objectstore-ec b/ceph/qa/suites/fs/multiclient/objectstore-ec new file mode 120000 index 000000000..a330d661a --- /dev/null +++ b/ceph/qa/suites/fs/multiclient/objectstore-ec @@ -0,0 +1 @@ +../../../cephfs/objectstore-ec/ \ No newline at end of file diff --git a/ceph/qa/suites/fs/multifs/clusters/2-remote-clients.yaml b/ceph/qa/suites/fs/multifs/clusters/2-remote-clients.yaml index 52c5d7e01..2ae772c3f 100644 --- a/ceph/qa/suites/fs/multifs/clusters/2-remote-clients.yaml +++ b/ceph/qa/suites/fs/multifs/clusters/2-remote-clients.yaml @@ -1,6 +1,6 @@ roles: -- [mon.a, mgr.x, osd.0, mon.b, mds.a, mds.b, client.1] -- [mds.c, mds.d, mon.c, client.0, osd.1, osd.2] +- [mon.a, mgr.x, osd.0, osd.1, osd.2, osd.3, mon.b, mds.a, mds.b, client.1] +- [mds.c, mds.d, mon.c, client.0, osd.4, osd.5, osd.6, osd.7] openstack: - volumes: # attached to each instance count: 2 diff --git a/ceph/qa/suites/fs/multifs/objectstore b/ceph/qa/suites/fs/multifs/objectstore deleted file mode 120000 index c72da2f82..000000000 --- a/ceph/qa/suites/fs/multifs/objectstore +++ /dev/null @@ -1 +0,0 @@ -../../../objectstore_cephfs \ No newline at end of file diff --git a/ceph/qa/suites/fs/multifs/objectstore-ec b/ceph/qa/suites/fs/multifs/objectstore-ec new file mode 120000 index 000000000..a330d661a --- /dev/null +++ b/ceph/qa/suites/fs/multifs/objectstore-ec @@ -0,0 +1 @@ +../../../cephfs/objectstore-ec/ \ No newline at end of file diff --git a/ceph/qa/suites/fs/permission/objectstore b/ceph/qa/suites/fs/permission/objectstore deleted file mode 120000 index c72da2f82..000000000 --- a/ceph/qa/suites/fs/permission/objectstore +++ /dev/null @@ -1 +0,0 @@ -../../../objectstore_cephfs \ No newline at end of file diff --git a/ceph/qa/suites/fs/permission/objectstore-ec b/ceph/qa/suites/fs/permission/objectstore-ec new file mode 120000 index 000000000..a330d661a --- /dev/null +++ b/ceph/qa/suites/fs/permission/objectstore-ec @@ -0,0 +1 @@ +../../../cephfs/objectstore-ec/ \ No newline at end of file diff --git a/ceph/qa/suites/fs/snaps/objectstore b/ceph/qa/suites/fs/snaps/objectstore deleted file mode 120000 index c72da2f82..000000000 --- a/ceph/qa/suites/fs/snaps/objectstore +++ /dev/null @@ -1 +0,0 @@ -../../../objectstore_cephfs \ No newline at end of file diff --git a/ceph/qa/suites/fs/snaps/objectstore-ec b/ceph/qa/suites/fs/snaps/objectstore-ec new file mode 120000 index 000000000..a330d661a --- /dev/null +++ b/ceph/qa/suites/fs/snaps/objectstore-ec @@ -0,0 +1 @@ +../../../cephfs/objectstore-ec/ \ No newline at end of file diff --git a/ceph/qa/suites/fs/thrash/objectstore b/ceph/qa/suites/fs/thrash/objectstore deleted file mode 120000 index c72da2f82..000000000 --- a/ceph/qa/suites/fs/thrash/objectstore +++ /dev/null @@ -1 +0,0 @@ -../../../objectstore_cephfs \ No newline at end of file diff --git a/ceph/qa/suites/fs/thrash/objectstore-ec b/ceph/qa/suites/fs/thrash/objectstore-ec new file mode 120000 index 000000000..a330d661a --- /dev/null +++ b/ceph/qa/suites/fs/thrash/objectstore-ec @@ -0,0 +1 @@ +../../../cephfs/objectstore-ec/ \ No newline at end of file diff --git a/ceph/qa/suites/fs/traceless/objectstore b/ceph/qa/suites/fs/traceless/objectstore deleted file mode 120000 index c72da2f82..000000000 --- a/ceph/qa/suites/fs/traceless/objectstore +++ /dev/null @@ -1 +0,0 @@ -../../../objectstore_cephfs \ No newline at end of file diff --git a/ceph/qa/suites/fs/traceless/objectstore-ec b/ceph/qa/suites/fs/traceless/objectstore-ec new file mode 120000 index 000000000..a330d661a --- /dev/null +++ b/ceph/qa/suites/fs/traceless/objectstore-ec @@ -0,0 +1 @@ +../../../cephfs/objectstore-ec/ \ No newline at end of file diff --git a/ceph/qa/suites/fs/verify/objectstore b/ceph/qa/suites/fs/verify/objectstore deleted file mode 120000 index c72da2f82..000000000 --- a/ceph/qa/suites/fs/verify/objectstore +++ /dev/null @@ -1 +0,0 @@ -../../../objectstore_cephfs \ No newline at end of file diff --git a/ceph/qa/suites/fs/verify/objectstore-ec b/ceph/qa/suites/fs/verify/objectstore-ec new file mode 120000 index 000000000..a330d661a --- /dev/null +++ b/ceph/qa/suites/fs/verify/objectstore-ec @@ -0,0 +1 @@ +../../../cephfs/objectstore-ec/ \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/cephfs/objectstore b/ceph/qa/suites/kcephfs/cephfs/objectstore deleted file mode 120000 index c72da2f82..000000000 --- a/ceph/qa/suites/kcephfs/cephfs/objectstore +++ /dev/null @@ -1 +0,0 @@ -../../../objectstore_cephfs \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/cephfs/objectstore-ec b/ceph/qa/suites/kcephfs/cephfs/objectstore-ec new file mode 120000 index 000000000..15dc98f23 --- /dev/null +++ b/ceph/qa/suites/kcephfs/cephfs/objectstore-ec @@ -0,0 +1 @@ +../../../cephfs/objectstore-ec \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/mixed-clients/objectstore b/ceph/qa/suites/kcephfs/mixed-clients/objectstore deleted file mode 120000 index c72da2f82..000000000 --- a/ceph/qa/suites/kcephfs/mixed-clients/objectstore +++ /dev/null @@ -1 +0,0 @@ -../../../objectstore_cephfs \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/mixed-clients/objectstore-ec b/ceph/qa/suites/kcephfs/mixed-clients/objectstore-ec new file mode 120000 index 000000000..15dc98f23 --- /dev/null +++ b/ceph/qa/suites/kcephfs/mixed-clients/objectstore-ec @@ -0,0 +1 @@ +../../../cephfs/objectstore-ec \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/recovery/clusters/4-remote-clients.yaml b/ceph/qa/suites/kcephfs/recovery/clusters/4-remote-clients.yaml index 1d432addb..b1072e3be 100644 --- a/ceph/qa/suites/kcephfs/recovery/clusters/4-remote-clients.yaml +++ b/ceph/qa/suites/kcephfs/recovery/clusters/4-remote-clients.yaml @@ -1,6 +1,6 @@ roles: -- [mon.a, osd.0, mds.a, mds.c, client.2] -- [mgr.x, osd.1, osd.2, mds.b, mds.d, client.3] +- [mon.a, osd.0, osd.1, osd.2, osd.3, mds.a, mds.c, client.2] +- [mgr.x, osd.4, osd.5, osd.6, osd.7, mds.b, mds.d, client.3] - [client.0] - [client.1] openstack: diff --git a/ceph/qa/suites/kcephfs/recovery/objectstore b/ceph/qa/suites/kcephfs/recovery/objectstore deleted file mode 120000 index c72da2f82..000000000 --- a/ceph/qa/suites/kcephfs/recovery/objectstore +++ /dev/null @@ -1 +0,0 @@ -../../../objectstore_cephfs \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/recovery/objectstore-ec b/ceph/qa/suites/kcephfs/recovery/objectstore-ec new file mode 120000 index 000000000..15dc98f23 --- /dev/null +++ b/ceph/qa/suites/kcephfs/recovery/objectstore-ec @@ -0,0 +1 @@ +../../../cephfs/objectstore-ec \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/thrash/objectstore b/ceph/qa/suites/kcephfs/thrash/objectstore deleted file mode 120000 index c72da2f82..000000000 --- a/ceph/qa/suites/kcephfs/thrash/objectstore +++ /dev/null @@ -1 +0,0 @@ -../../../objectstore_cephfs \ No newline at end of file diff --git a/ceph/qa/suites/kcephfs/thrash/objectstore-ec b/ceph/qa/suites/kcephfs/thrash/objectstore-ec new file mode 120000 index 000000000..15dc98f23 --- /dev/null +++ b/ceph/qa/suites/kcephfs/thrash/objectstore-ec @@ -0,0 +1 @@ +../../../cephfs/objectstore-ec \ No newline at end of file diff --git a/ceph/qa/suites/multimds/basic/objectstore b/ceph/qa/suites/multimds/basic/objectstore deleted file mode 120000 index c72da2f82..000000000 --- a/ceph/qa/suites/multimds/basic/objectstore +++ /dev/null @@ -1 +0,0 @@ -../../../objectstore_cephfs \ No newline at end of file diff --git a/ceph/qa/suites/multimds/basic/objectstore-ec b/ceph/qa/suites/multimds/basic/objectstore-ec new file mode 120000 index 000000000..15dc98f23 --- /dev/null +++ b/ceph/qa/suites/multimds/basic/objectstore-ec @@ -0,0 +1 @@ +../../../cephfs/objectstore-ec \ No newline at end of file diff --git a/ceph/qa/suites/multimds/thrash/objectstore b/ceph/qa/suites/multimds/thrash/objectstore deleted file mode 120000 index c72da2f82..000000000 --- a/ceph/qa/suites/multimds/thrash/objectstore +++ /dev/null @@ -1 +0,0 @@ -../../../objectstore_cephfs \ No newline at end of file diff --git a/ceph/qa/suites/multimds/thrash/objectstore-ec b/ceph/qa/suites/multimds/thrash/objectstore-ec new file mode 120000 index 000000000..15dc98f23 --- /dev/null +++ b/ceph/qa/suites/multimds/thrash/objectstore-ec @@ -0,0 +1 @@ +../../../cephfs/objectstore-ec \ No newline at end of file diff --git a/ceph/qa/suites/multimds/verify/objectstore b/ceph/qa/suites/multimds/verify/objectstore deleted file mode 120000 index c72da2f82..000000000 --- a/ceph/qa/suites/multimds/verify/objectstore +++ /dev/null @@ -1 +0,0 @@ -../../../objectstore_cephfs \ No newline at end of file diff --git a/ceph/qa/suites/multimds/verify/objectstore-ec b/ceph/qa/suites/multimds/verify/objectstore-ec new file mode 120000 index 000000000..15dc98f23 --- /dev/null +++ b/ceph/qa/suites/multimds/verify/objectstore-ec @@ -0,0 +1 @@ +../../../cephfs/objectstore-ec \ No newline at end of file diff --git a/ceph/qa/suites/rados/basic/d-require-luminous b/ceph/qa/suites/rados/basic/d-require-luminous deleted file mode 120000 index 737aee824..000000000 --- a/ceph/qa/suites/rados/basic/d-require-luminous +++ /dev/null @@ -1 +0,0 @@ -../thrash/d-require-luminous/ \ No newline at end of file diff --git a/ceph/qa/suites/rados/basic/d-require-luminous/at-end.yaml b/ceph/qa/suites/rados/basic/d-require-luminous/at-end.yaml new file mode 100644 index 000000000..ef998cc89 --- /dev/null +++ b/ceph/qa/suites/rados/basic/d-require-luminous/at-end.yaml @@ -0,0 +1,33 @@ +# do not require luminous osds at mkfs time; only set flag at +# the end of the test run, then do a final scrub (to convert any +# legacy snapsets), and verify we are healthy. +tasks: +- full_sequential_finally: + - exec: + mon.a: + - ceph osd require-osd-release luminous + - ceph osd pool application enable base rados || true +# make sure osds have latest map + - rados -p rbd bench 5 write -b 4096 + - ceph.healthy: + - ceph.osd_scrub_pgs: + cluster: ceph + - exec: + mon.a: + - sleep 15 + - ceph osd dump | grep purged_snapdirs + - ceph pg dump -f json-pretty + - "ceph pg dump sum -f json-pretty | grep num_legacy_snapsets | head -1 | grep ': 0'" +overrides: + ceph: + conf: + global: + mon debug no require luminous: true + +# setting luminous triggers peering, which *might* trigger health alerts + log-whitelist: + - overall HEALTH_ + - \(PG_AVAILABILITY\) + - \(PG_DEGRADED\) + thrashosds: + chance_thrash_cluster_full: 0 diff --git a/ceph/qa/suites/upgrade/jewel-x/parallel/5-final-workload/+ b/ceph/qa/suites/rados/basic/d-require-luminous/at-mkfs.yaml similarity index 100% rename from ceph/qa/suites/upgrade/jewel-x/parallel/5-final-workload/+ rename to ceph/qa/suites/rados/basic/d-require-luminous/at-mkfs.yaml diff --git a/ceph/qa/suites/rados/mgr/clusters/2-node-mgr.yaml b/ceph/qa/suites/rados/mgr/clusters/2-node-mgr.yaml index bc950e5af..abc90e22d 100644 --- a/ceph/qa/suites/rados/mgr/clusters/2-node-mgr.yaml +++ b/ceph/qa/suites/rados/mgr/clusters/2-node-mgr.yaml @@ -1,6 +1,6 @@ roles: - [mgr.x, mon.a, mon.c, mds.a, mds.c, osd.0, client.0] -- [mgr.y, mon.b, mds.b, osd.1, osd.2, client.1] +- [mgr.y, mgr.z, mon.b, mds.b, osd.1, osd.2, client.1] log-rotate: ceph-mds: 10G ceph-osd: 10G diff --git a/ceph/qa/suites/rados/mgr/tasks/dashboard.yaml b/ceph/qa/suites/rados/mgr/tasks/dashboard.yaml new file mode 100644 index 000000000..3065e11bc --- /dev/null +++ b/ceph/qa/suites/rados/mgr/tasks/dashboard.yaml @@ -0,0 +1,16 @@ + +tasks: + - install: + - ceph: + # tests may leave mgrs broken, so don't try and call into them + # to invoke e.g. pg dump during teardown. + wait-for-scrub: false + log-whitelist: + - overall HEALTH_ + - \(MGR_DOWN\) + - \(PG_ + - replacing it with standby + - No standby daemons available + - cephfs_test_runner: + modules: + - tasks.mgr.test_dashboard diff --git a/ceph/qa/suites/rados/mgr/tasks/module_selftest.yaml b/ceph/qa/suites/rados/mgr/tasks/module_selftest.yaml new file mode 100644 index 000000000..ffdfe8be2 --- /dev/null +++ b/ceph/qa/suites/rados/mgr/tasks/module_selftest.yaml @@ -0,0 +1,19 @@ + +tasks: + - install: + - ceph: + # tests may leave mgrs broken, so don't try and call into them + # to invoke e.g. pg dump during teardown. + wait-for-scrub: false + log-whitelist: + - overall HEALTH_ + - \(MGR_DOWN\) + - \(PG_ + - replacing it with standby + - No standby daemons available + - Reduced data availability + - Degraded data redundancy + - objects misplaced + - cephfs_test_runner: + modules: + - tasks.mgr.test_module_selftest diff --git a/ceph/qa/suites/rados/mgr/tasks/workunits.yaml b/ceph/qa/suites/rados/mgr/tasks/workunits.yaml new file mode 100644 index 000000000..d7261f44b --- /dev/null +++ b/ceph/qa/suites/rados/mgr/tasks/workunits.yaml @@ -0,0 +1,16 @@ +tasks: + - install: + - ceph: + # tests may leave mgrs broken, so don't try and call into them + # to invoke e.g. pg dump during teardown. + wait-for-scrub: false + log-whitelist: + - overall HEALTH_ + - \(MGR_DOWN\) + - \(PG_ + - replacing it with standby + - No standby daemons available + - workunit: + clients: + client.0: + - mgr \ No newline at end of file diff --git a/ceph/qa/suites/rados/monthrash/d-require-luminous b/ceph/qa/suites/rados/monthrash/d-require-luminous index 737aee824..82036c67f 120000 --- a/ceph/qa/suites/rados/monthrash/d-require-luminous +++ b/ceph/qa/suites/rados/monthrash/d-require-luminous @@ -1 +1 @@ -../thrash/d-require-luminous/ \ No newline at end of file +../basic/d-require-luminous \ No newline at end of file diff --git a/ceph/qa/suites/rados/rest/mgr-restful.yaml b/ceph/qa/suites/rados/rest/mgr-restful.yaml index 90906d66e..049532e34 100644 --- a/ceph/qa/suites/rados/rest/mgr-restful.yaml +++ b/ceph/qa/suites/rados/rest/mgr-restful.yaml @@ -6,6 +6,9 @@ tasks: log-whitelist: - overall HEALTH_ - \(MGR_DOWN\) + - \(PG_ + - \(OSD_ + - \(OBJECT_ - exec: mon.a: - ceph restful create-key admin diff --git a/ceph/qa/suites/rest/basic/tasks/rest_test.yaml b/ceph/qa/suites/rados/rest/rest_test.yaml similarity index 62% rename from ceph/qa/suites/rest/basic/tasks/rest_test.yaml rename to ceph/qa/suites/rados/rest/rest_test.yaml index 948545623..0fdb9dc6a 100644 --- a/ceph/qa/suites/rest/basic/tasks/rest_test.yaml +++ b/ceph/qa/suites/rados/rest/rest_test.yaml @@ -20,7 +20,18 @@ tasks: - ceph: fs: xfs log-whitelist: - - but it is still running + - overall HEALTH + - \(OSDMAP_FLAGS\) + - \(OSD_ + - \(PG_ + - \(POOL_ + - \(CACHE_POOL_ + - \(SMALLER_PGP_NUM\) + - \(OBJECT_ + - \(REQUEST_SLOW\) + - \(SLOW_OPS\) + - \(TOO_FEW_PGS\) + - but it is still running conf: client.rest0: debug ms: 1 diff --git a/ceph/qa/suites/rados/singleton-nomsgr/all/admin_socket_output.yaml b/ceph/qa/suites/rados/singleton-nomsgr/all/admin_socket_output.yaml index 3aaca8759..bbf330b0b 100644 --- a/ceph/qa/suites/rados/singleton-nomsgr/all/admin_socket_output.yaml +++ b/ceph/qa/suites/rados/singleton-nomsgr/all/admin_socket_output.yaml @@ -9,6 +9,7 @@ overrides: - (OSDMAP_FLAGS) - (OSD_FULL) - (MDS_READ_ONLY) + - (POOL_FULL) tasks: - install: - ceph: diff --git a/ceph/qa/suites/rados/singleton/all/max-pg-per-osd.from-mon.yaml b/ceph/qa/suites/rados/singleton/all/max-pg-per-osd.from-mon.yaml new file mode 100644 index 000000000..accdd964f --- /dev/null +++ b/ceph/qa/suites/rados/singleton/all/max-pg-per-osd.from-mon.yaml @@ -0,0 +1,26 @@ +roles: +- - mon.a + - mgr.x + - osd.0 + - osd.1 +openstack: + - volumes: # attached to each instance + count: 2 + size: 10 # GB +overrides: + ceph: + create_rbd_pool: False + conf: + mon: + osd pool default size: 2 + osd: + mon max pg per osd : 2 + osd max pg per osd hard ratio : 1 + log-whitelist: + - \(TOO_FEW_PGS\) +tasks: +- install: +- ceph: +- osd_max_pg_per_osd: + test_create_from_mon: True + pg_num: 2 diff --git a/ceph/qa/suites/rados/singleton/all/max-pg-per-osd.from-primary.yaml b/ceph/qa/suites/rados/singleton/all/max-pg-per-osd.from-primary.yaml new file mode 100644 index 000000000..1c48ada75 --- /dev/null +++ b/ceph/qa/suites/rados/singleton/all/max-pg-per-osd.from-primary.yaml @@ -0,0 +1,31 @@ +roles: +- - mon.a + - mgr.x + - osd.0 + - osd.1 + - osd.2 + - osd.3 +openstack: + - volumes: # attached to each instance + count: 4 + size: 10 # GB +overrides: + ceph: + create_rbd_pool: False + conf: + mon: + osd pool default size: 2 + osd: + mon max pg per osd : 1 + osd max pg per osd hard ratio : 1 + log-whitelist: + - \(TOO_FEW_PGS\) + - \(PG_ +tasks: +- install: +- ceph: +- osd_max_pg_per_osd: + test_create_from_mon: False + pg_num: 1 + pool_size: 2 + from_primary: True diff --git a/ceph/qa/suites/rados/singleton/all/max-pg-per-osd.from-replica.yaml b/ceph/qa/suites/rados/singleton/all/max-pg-per-osd.from-replica.yaml new file mode 100644 index 000000000..0cf37fd8e --- /dev/null +++ b/ceph/qa/suites/rados/singleton/all/max-pg-per-osd.from-replica.yaml @@ -0,0 +1,31 @@ +roles: +- - mon.a + - mgr.x + - osd.0 + - osd.1 + - osd.2 + - osd.3 +openstack: + - volumes: # attached to each instance + count: 4 + size: 10 # GB +overrides: + ceph: + create_rbd_pool: False + conf: + mon: + osd pool default size: 2 + osd: + mon max pg per osd : 1 + osd max pg per osd hard ratio : 1 + log-whitelist: + - \(TOO_FEW_PGS\) + - \(PG_ +tasks: +- install: +- ceph: +- osd_max_pg_per_osd: + test_create_from_mon: False + pg_num: 1 + pool_size: 2 + from_primary: False diff --git a/ceph/qa/suites/rados/singleton/all/mon-seesaw.yaml b/ceph/qa/suites/rados/singleton/all/mon-seesaw.yaml index ccd980fde..815c518ee 100644 --- a/ceph/qa/suites/rados/singleton/all/mon-seesaw.yaml +++ b/ceph/qa/suites/rados/singleton/all/mon-seesaw.yaml @@ -17,6 +17,10 @@ tasks: osd: debug monc: 1 debug ms: 1 + log-whitelist: + - overall HEALTH + - Manager daemon + - \(MGR_DOWN\) - mon_seesaw: - ceph_manager.create_pool: kwargs: diff --git a/ceph/qa/suites/rados/singleton/all/recovery-preemption.yaml b/ceph/qa/suites/rados/singleton/all/recovery-preemption.yaml new file mode 100644 index 000000000..7507bf635 --- /dev/null +++ b/ceph/qa/suites/rados/singleton/all/recovery-preemption.yaml @@ -0,0 +1,51 @@ +roles: +- - mon.a + - mon.b + - mon.c + - mgr.x + - osd.0 + - osd.1 + - osd.2 + - osd.3 +openstack: + - volumes: # attached to each instance + count: 3 + size: 20 # GB +tasks: +- install: +- ceph: + conf: + osd: + osd recovery sleep: .1 + osd min pg log entries: 100 + osd max pg log entries: 1000 + log-whitelist: + - \(POOL_APP_NOT_ENABLED\) + - \(OSDMAP_FLAGS\) + - \(OSD_ + - \(OBJECT_ + - \(PG_ + - overall HEALTH +- exec: + osd.0: + - ceph osd pool create foo 128 + - ceph osd pool application enable foo foo + - rados -p foo bench 30 write -b 4096 --no-cleanup + - ceph osd out 0 + - sleep 5 + - ceph osd set noup +- ceph.restart: + daemons: [osd.1] + wait-for-up: false + wait-for-healthy: false +- exec: + osd.0: + - rados -p foo bench 3 write -b 4096 --no-cleanup + - ceph osd unset noup + - sleep 10 + - ceph tell osd.* config set osd_recovery_sleep 0 + - ceph tell osd.* config set osd_recovery_max_active 20 +- ceph.healthy: +- exec: + osd.0: + - egrep '(defer backfill|defer recovery)' /var/log/ceph/ceph-osd.*.log diff --git a/ceph/qa/suites/rados/thrash/d-require-luminous/at-mkfs-balancer-crush-compat.yaml b/ceph/qa/suites/rados/thrash/d-require-luminous/at-mkfs-balancer-crush-compat.yaml new file mode 100644 index 000000000..9eb7143de --- /dev/null +++ b/ceph/qa/suites/rados/thrash/d-require-luminous/at-mkfs-balancer-crush-compat.yaml @@ -0,0 +1,11 @@ +overrides: + ceph: + conf: + mgr: + debug osd: 20 +tasks: +- exec: + mon.a: + - while ! ceph balancer status ; do sleep 1 ; done + - ceph balancer mode crush-compat + - ceph balancer on diff --git a/ceph/qa/suites/rados/thrash/d-require-luminous/at-mkfs-balancer-upmap.yaml b/ceph/qa/suites/rados/thrash/d-require-luminous/at-mkfs-balancer-upmap.yaml new file mode 100644 index 000000000..a1e0afea0 --- /dev/null +++ b/ceph/qa/suites/rados/thrash/d-require-luminous/at-mkfs-balancer-upmap.yaml @@ -0,0 +1,11 @@ +overrides: + ceph: + conf: + mgr: + debug osd: 20 +tasks: +- exec: + mon.a: + - while ! ceph balancer status ; do sleep 1 ; done + - ceph balancer mode upmap + - ceph balancer on diff --git a/ceph/qa/suites/rados/verify/d-require-luminous b/ceph/qa/suites/rados/verify/d-require-luminous index 737aee824..82036c67f 120000 --- a/ceph/qa/suites/rados/verify/d-require-luminous +++ b/ceph/qa/suites/rados/verify/d-require-luminous @@ -1 +1 @@ -../thrash/d-require-luminous/ \ No newline at end of file +../basic/d-require-luminous \ No newline at end of file diff --git a/ceph/qa/suites/rbd/basic/tasks/rbd_cls_tests.yaml b/ceph/qa/suites/rbd/basic/tasks/rbd_cls_tests.yaml index 9ccd57c4a..51b35e2e1 100644 --- a/ceph/qa/suites/rbd/basic/tasks/rbd_cls_tests.yaml +++ b/ceph/qa/suites/rbd/basic/tasks/rbd_cls_tests.yaml @@ -3,3 +3,5 @@ tasks: clients: client.0: - cls/test_cls_rbd.sh + - cls/test_cls_lock.sh + - cls/test_cls_journal.sh diff --git a/ceph/qa/suites/rgw/hadoop-s3a/s3a-hadoop.yaml b/ceph/qa/suites/rgw/hadoop-s3a/s3a-hadoop.yaml index 171cc66e2..1c17a69f7 100644 --- a/ceph/qa/suites/rgw/hadoop-s3a/s3a-hadoop.yaml +++ b/ceph/qa/suites/rgw/hadoop-s3a/s3a-hadoop.yaml @@ -1,6 +1,8 @@ -os_type: centos -os_version: "7.3" -machine_type: vps +machine_type: ovh +openstack: +- volumes: # attached to each instance + count: 3 + size: 10 # GB overrides: ceph_ansible: vars: @@ -9,15 +11,16 @@ overrides: osd default pool size: 2 osd pool default pg num: 128 osd pool default pgp num: 128 - debug rgw: 20 + debug rgw: 20 debug ms: 1 ceph_test: true - ceph_dev: true - ceph_dev_key: https://download.ceph.com/keys/autobuild.asc - ceph_origin: upstream journal_collocation: true osd_auto_discovery: false journal_size: 1024 + ceph_stable_release: luminous + osd_scenario: collocated + ceph_origin: repository + ceph_repository: dev roles: - [mon.a, osd.0, osd.1, osd.2, rgw.0] - [osd.3, osd.4, osd.5] diff --git a/ceph/qa/suites/rgw/multifs/tasks/rgw_s3tests.yaml b/ceph/qa/suites/rgw/multifs/tasks/rgw_s3tests.yaml index 4cdded04e..da05a5ea1 100644 --- a/ceph/qa/suites/rgw/multifs/tasks/rgw_s3tests.yaml +++ b/ceph/qa/suites/rgw/multifs/tasks/rgw_s3tests.yaml @@ -4,7 +4,7 @@ tasks: - rgw: [client.0] - s3tests: client.0: - force-branch: ceph-master + force-branch: ceph-luminous rgw_server: client.0 overrides: ceph: diff --git a/ceph/qa/suites/rgw/thrash/workload/rgw_s3tests.yaml b/ceph/qa/suites/rgw/thrash/workload/rgw_s3tests.yaml index 45047ea41..82ac7c197 100644 --- a/ceph/qa/suites/rgw/thrash/workload/rgw_s3tests.yaml +++ b/ceph/qa/suites/rgw/thrash/workload/rgw_s3tests.yaml @@ -1,7 +1,7 @@ tasks: - s3tests: client.0: - force-branch: ceph-master + force-branch: ceph-luminous rgw_server: client.0 overrides: ceph: diff --git a/ceph/qa/suites/rgw/verify/tasks/rgw_s3tests.yaml b/ceph/qa/suites/rgw/verify/tasks/rgw_s3tests.yaml index bed9f0e19..cf413389b 100644 --- a/ceph/qa/suites/rgw/verify/tasks/rgw_s3tests.yaml +++ b/ceph/qa/suites/rgw/verify/tasks/rgw_s3tests.yaml @@ -10,7 +10,7 @@ tasks: valgrind: [--tool=memcheck] - s3tests: client.0: - force-branch: ceph-master + force-branch: ceph-luminous rgw_server: client.0 overrides: ceph: diff --git a/ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/+ b/ceph/qa/suites/upgrade/jewel-x/ceph-deploy/% similarity index 100% rename from ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/+ rename to ceph/qa/suites/upgrade/jewel-x/ceph-deploy/% diff --git a/ceph/qa/suites/upgrade/jewel-x/ceph-deploy/distros/centos_latest.yaml b/ceph/qa/suites/upgrade/jewel-x/ceph-deploy/distros/centos_latest.yaml new file mode 120000 index 000000000..b5973b952 --- /dev/null +++ b/ceph/qa/suites/upgrade/jewel-x/ceph-deploy/distros/centos_latest.yaml @@ -0,0 +1 @@ +../../../../../distros/supported/centos_latest.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/jewel-x/ceph-deploy/distros/ubuntu_latest.yaml b/ceph/qa/suites/upgrade/jewel-x/ceph-deploy/distros/ubuntu_latest.yaml new file mode 120000 index 000000000..cc5b15bcc --- /dev/null +++ b/ceph/qa/suites/upgrade/jewel-x/ceph-deploy/distros/ubuntu_latest.yaml @@ -0,0 +1 @@ +../../../../../distros/supported/ubuntu_latest.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/jewel-x/ceph-deploy/jewel-luminous.yaml b/ceph/qa/suites/upgrade/jewel-x/ceph-deploy/jewel-luminous.yaml new file mode 100644 index 000000000..9adede74f --- /dev/null +++ b/ceph/qa/suites/upgrade/jewel-x/ceph-deploy/jewel-luminous.yaml @@ -0,0 +1,82 @@ +meta: +- desc: | + Setup 4 node ceph cluster using ceph-deploy, use latest + stable jewel as initial release, upgrade to luminous and + also setup mgr nodes along after upgrade, check for + cluster to reach healthy state, After upgrade run kernel tar/untar + task and systemd task. This test will detect any + ceph upgrade issue and systemd issues. +overrides: + ceph-deploy: + fs: xfs + conf: + global: + mon pg warn min per osd: 2 + osd: + osd pool default size: 2 + osd objectstore: filestore + osd sloppy crc: true + client: + rbd default features: 5 +openstack: +- machine: + disk: 100 +- volumes: + count: 3 + size: 30 +# reluctantely :( hard-coded machine type +# it will override command line args with teuthology-suite +machine_type: vps +roles: +- - mon.a + - mds.a + - osd.0 + - osd.1 + - osd.2 + - mgr.x +- - mon.b + - mgr.y +- - mon.c + - osd.3 + - osd.4 + - osd.5 +- - osd.6 + - osd.7 + - osd.8 + - client.0 +tasks: +- ssh-keys: +- print: "**** done ssh-keys" +- ceph-deploy: + branch: + stable: jewel + skip-mgr: True +- print: "**** done initial ceph-deploy" +- ceph-deploy.upgrade: + branch: + dev: luminous + setup-mgr-node: True + check-for-healthy: True + roles: + - mon.a + - mon.b + - mon.c + - osd.6 +- print: "**** done ceph-deploy upgrade" +- exec: + osd.0: + - ceph osd require-osd-release luminous + - ceph osd set-require-min-compat-client luminous +- print: "**** done `ceph osd require-osd-release luminous`" +- workunit: + clients: + all: + - kernel_untar_build.sh +- print: "**** done kernel_untar_build.sh" +- systemd: +- print: "**** done systemd" +- workunit: + clients: + all: + - rados/load-gen-mix.sh +- print: "**** done rados/load-gen-mix.sh" diff --git a/ceph/qa/suites/upgrade/jewel-x/parallel/0-cluster/start.yaml b/ceph/qa/suites/upgrade/jewel-x/parallel/0-cluster/start.yaml index 314562632..d1f1e1070 100644 --- a/ceph/qa/suites/upgrade/jewel-x/parallel/0-cluster/start.yaml +++ b/ceph/qa/suites/upgrade/jewel-x/parallel/0-cluster/start.yaml @@ -18,6 +18,7 @@ roles: - client.1 - client.2 - client.3 +- - client.4 overrides: ceph: log-whitelist: diff --git a/ceph/qa/suites/upgrade/jewel-x/parallel/1-jewel-install/jewel.yaml b/ceph/qa/suites/upgrade/jewel-x/parallel/1-jewel-install/jewel.yaml index a367ef37c..c64b2cded 100644 --- a/ceph/qa/suites/upgrade/jewel-x/parallel/1-jewel-install/jewel.yaml +++ b/ceph/qa/suites/upgrade/jewel-x/parallel/1-jewel-install/jewel.yaml @@ -1,3 +1,22 @@ +overrides: + ceph: + conf: + client.0: + debug ms: 1 + debug client: 10 + debug monc: 10 + client.1: + debug ms: 1 + debug client: 10 + debug monc: 10 + client.2: + debug ms: 1 + debug client: 10 + debug monc: 10 + client.3: + debug ms: 1 + debug client: 10 + debug monc: 10 meta: - desc: | install ceph/jewel latest diff --git a/ceph/qa/suites/upgrade/jewel-x/parallel/2-workload/blogbench.yaml b/ceph/qa/suites/upgrade/jewel-x/parallel/2-workload/blogbench.yaml index a8e28c52c..56eedbd6b 100644 --- a/ceph/qa/suites/upgrade/jewel-x/parallel/2-workload/blogbench.yaml +++ b/ceph/qa/suites/upgrade/jewel-x/parallel/2-workload/blogbench.yaml @@ -5,7 +5,7 @@ meta: workload: full_sequential: - sequential: - - ceph-fuse: + - ceph-fuse: [client.2] - print: "**** done ceph-fuse 2-workload" - workunit: clients: diff --git a/ceph/qa/suites/upgrade/jewel-x/parallel/4-luminous.yaml b/ceph/qa/suites/upgrade/jewel-x/parallel/4-luminous.yaml deleted file mode 120000 index 5283ac73e..000000000 --- a/ceph/qa/suites/upgrade/jewel-x/parallel/4-luminous.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../../releases/luminous.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/jewel-x/parallel/4-luminous.yaml b/ceph/qa/suites/upgrade/jewel-x/parallel/4-luminous.yaml new file mode 100644 index 000000000..e57b37753 --- /dev/null +++ b/ceph/qa/suites/upgrade/jewel-x/parallel/4-luminous.yaml @@ -0,0 +1,23 @@ +# this is the same fragment as ../../../../releases/luminous.yaml +# but without line "ceph osd set-require-min-compat-client luminous" + +tasks: +- exec: + mgr.x: + - mkdir -p /var/lib/ceph/mgr/ceph-x + - ceph auth get-or-create-key mgr.x mon 'allow profile mgr' + - ceph auth export mgr.x > /var/lib/ceph/mgr/ceph-x/keyring +- ceph.restart: + daemons: [mgr.x] + wait-for-healthy: false +- exec: + osd.0: + - ceph osd require-osd-release luminous +- ceph.healthy: +overrides: + ceph: + conf: + mon: + mon warn on osd down out interval zero: false + log-whitelist: + - no active mgr diff --git a/ceph/qa/suites/upgrade/jewel-x/parallel/5-workload.yaml b/ceph/qa/suites/upgrade/jewel-x/parallel/5-workload.yaml new file mode 100644 index 000000000..f7e9de46a --- /dev/null +++ b/ceph/qa/suites/upgrade/jewel-x/parallel/5-workload.yaml @@ -0,0 +1,11 @@ +meta: +- desc: | + run basic import/export cli tests for rbd on not upgrated client.4 + (covers issue http://tracker.ceph.com/issues/21660) +tasks: + - workunit: + branch: jewel + clients: + client.4: + - rbd/import_export.sh + - print: "**** done rbd/import_export.sh 5-workload" diff --git a/ceph/qa/suites/upgrade/kraken-x/parallel/4-luminous-with-mgr.yaml b/ceph/qa/suites/upgrade/jewel-x/parallel/6-luminous-with-mgr.yaml similarity index 100% rename from ceph/qa/suites/upgrade/kraken-x/parallel/4-luminous-with-mgr.yaml rename to ceph/qa/suites/upgrade/jewel-x/parallel/6-luminous-with-mgr.yaml diff --git a/ceph/qa/suites/upgrade/jewel-x/parallel/6.5-crush-compat.yaml b/ceph/qa/suites/upgrade/jewel-x/parallel/6.5-crush-compat.yaml new file mode 100644 index 000000000..20c0ffd9f --- /dev/null +++ b/ceph/qa/suites/upgrade/jewel-x/parallel/6.5-crush-compat.yaml @@ -0,0 +1,8 @@ +tasks: +- exec: + mon.a: + - ceph osd set-require-min-compat-client jewel + - ceph osd crush set-all-straw-buckets-to-straw2 + - ceph osd crush weight-set create-compat + - ceph osd crush weight-set reweight-compat osd.0 .9 + - ceph osd crush weight-set reweight-compat osd.1 1.2 diff --git a/ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/+ b/ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/+ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/blogbench.yaml b/ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/blogbench.yaml similarity index 74% rename from ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/blogbench.yaml rename to ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/blogbench.yaml index d2629c03f..d73459e43 100644 --- a/ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/blogbench.yaml +++ b/ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/blogbench.yaml @@ -4,10 +4,10 @@ meta: mount ceph-fuse on client.3 before running workunit tasks: - sequential: - - ceph-fuse: + - ceph-fuse: [client.3] - print: "**** done ceph-fuse 5-final-workload" - workunit: clients: client.3: - suites/blogbench.sh - - print: "**** done suites/blogbench.sh 5-final-workload" + - print: "**** done suites/blogbench.sh 7-final-workload" diff --git a/ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/rados-snaps-few-objects.yaml b/ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/rados-snaps-few-objects.yaml similarity index 88% rename from ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/rados-snaps-few-objects.yaml rename to ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/rados-snaps-few-objects.yaml index d8b3dcb38..7dd61c5fc 100644 --- a/ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/rados-snaps-few-objects.yaml +++ b/ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/rados-snaps-few-objects.yaml @@ -14,4 +14,4 @@ tasks: snap_create: 50 snap_remove: 50 rollback: 50 - - print: "**** done rados 4-final-workload" + - print: "**** done rados 7-final-workload" diff --git a/ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/rados_loadgenmix.yaml b/ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/rados_loadgenmix.yaml similarity index 74% rename from ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/rados_loadgenmix.yaml rename to ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/rados_loadgenmix.yaml index 922a9da4f..b218b9226 100644 --- a/ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/rados_loadgenmix.yaml +++ b/ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/rados_loadgenmix.yaml @@ -6,4 +6,4 @@ tasks: clients: client.1: - rados/load-gen-mix.sh - - print: "**** done rados/load-gen-mix.sh 4-final-workload" + - print: "**** done rados/load-gen-mix.sh 7-final-workload" diff --git a/ceph/qa/suites/upgrade/jewel-x/parallel/5-final-workload/rados_mon_thrash.yaml b/ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/rados_mon_thrash.yaml similarity index 84% rename from ceph/qa/suites/upgrade/jewel-x/parallel/5-final-workload/rados_mon_thrash.yaml rename to ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/rados_mon_thrash.yaml index 9b60d2ebc..c835a659b 100644 --- a/ceph/qa/suites/upgrade/jewel-x/parallel/5-final-workload/rados_mon_thrash.yaml +++ b/ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/rados_mon_thrash.yaml @@ -15,4 +15,4 @@ tasks: clients: client.1: - rados/test-upgrade-v11.0.0.sh - - print: "**** done rados/test-upgrade-v11.0.0.sh 4-final-workload" + - print: "**** done rados/test-upgrade-v11.0.0.sh 7-final-workload" diff --git a/ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/rbd_cls.yaml b/ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/rbd_cls.yaml similarity index 69% rename from ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/rbd_cls.yaml rename to ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/rbd_cls.yaml index aaf0a3779..46bbf7610 100644 --- a/ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/rbd_cls.yaml +++ b/ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/rbd_cls.yaml @@ -6,4 +6,4 @@ tasks: clients: client.1: - cls/test_cls_rbd.sh - - print: "**** done cls/test_cls_rbd.sh 4-final-workload" + - print: "**** done cls/test_cls_rbd.sh 7-final-workload" diff --git a/ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/rbd_import_export.yaml b/ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/rbd_import_export.yaml similarity index 76% rename from ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/rbd_import_export.yaml rename to ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/rbd_import_export.yaml index 46e135506..5ae749188 100644 --- a/ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/rbd_import_export.yaml +++ b/ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/rbd_import_export.yaml @@ -8,4 +8,4 @@ tasks: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - - print: "**** done rbd/import_export.sh 4-final-workload" + - print: "**** done rbd/import_export.sh 7-final-workload" diff --git a/ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/rgw_swift.yaml b/ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/rgw_swift.yaml similarity index 64% rename from ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/rgw_swift.yaml rename to ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/rgw_swift.yaml index 7a7659ff4..780c4ad70 100644 --- a/ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/rgw_swift.yaml +++ b/ceph/qa/suites/upgrade/jewel-x/parallel/7-final-workload/rgw_swift.yaml @@ -6,8 +6,8 @@ overrides: frontend: civetweb tasks: - rgw: [client.1] - - print: "**** done rgw 4-final-workload" + - print: "**** done rgw 7-final-workload" - swift: client.1: rgw_server: client.1 - - print: "**** done swift 4-final-workload" + - print: "**** done swift 7-final-workload" diff --git a/ceph/qa/suites/upgrade/jewel-x/parallel/8-jewel-workload.yaml b/ceph/qa/suites/upgrade/jewel-x/parallel/8-jewel-workload.yaml new file mode 120000 index 000000000..81df389c3 --- /dev/null +++ b/ceph/qa/suites/upgrade/jewel-x/parallel/8-jewel-workload.yaml @@ -0,0 +1 @@ +5-workload.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/jewel-x/point-to-point-x/point-to-point-upgrade.yaml b/ceph/qa/suites/upgrade/jewel-x/point-to-point-x/point-to-point-upgrade.yaml index 3033f14be..d68c258c0 100644 --- a/ceph/qa/suites/upgrade/jewel-x/point-to-point-x/point-to-point-upgrade.yaml +++ b/ceph/qa/suites/upgrade/jewel-x/point-to-point-x/point-to-point-upgrade.yaml @@ -16,13 +16,20 @@ overrides: - scrub - osd_map_max_advance - wrongly marked + - overall HEALTH_ + - \(MGR_DOWN\) + - \(OSD_ + - \(PG_ + - \(CACHE_ fs: xfs conf: + global: + mon warn on pool no app: false mon: mon debug unsafe allow tier with nonempty snaps: true - mon warn on pool no app: false osd: osd map max advance: 1000 + osd map cache size: 1100 roles: - - mon.a - mds.a @@ -161,7 +168,7 @@ workload_x: branch: jewel clients: client.1: - - rados/test-upgrade-v11.0.0.sh + - rados/test-upgrade-v11.0.0-noec.sh - cls env: CLS_RBD_GTEST_FILTER: '*:-TestClsRbd.mirror_image' @@ -170,7 +177,7 @@ workload_x: branch: jewel clients: client.0: - - rados/test-upgrade-v11.0.0.sh + - rados/test-upgrade-v11.0.0-noec.sh - cls - print: "**** done rados/test-upgrade-v11.0.0.sh & cls workload_x upgraded client" - rgw: [client.1] diff --git a/ceph/qa/suites/upgrade/jewel-x/stress-split/6.5-crush-compat.yaml b/ceph/qa/suites/upgrade/jewel-x/stress-split/6.5-crush-compat.yaml new file mode 120000 index 000000000..02263d105 --- /dev/null +++ b/ceph/qa/suites/upgrade/jewel-x/stress-split/6.5-crush-compat.yaml @@ -0,0 +1 @@ +../parallel/6.5-crush-compat.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/kraken-x/ceph-deploy/kraken-luminous.yaml b/ceph/qa/suites/upgrade/kraken-x/ceph-deploy/kraken-luminous.yaml new file mode 100644 index 000000000..4a5536275 --- /dev/null +++ b/ceph/qa/suites/upgrade/kraken-x/ceph-deploy/kraken-luminous.yaml @@ -0,0 +1,61 @@ +meta: +- desc: | + Setup 4 node ceph cluster using ceph-deploy, use latest + stable kraken as initial release, upgrade to luminous and + also setup mgr nodes along after upgrade, check for + cluster to reach healthy state, After upgrade run kernel tar/untar + task and systemd task. This test will detect any + ceph upgrade issue and systemd issues. +overrides: + ceph-deploy: + fs: xfs + conf: + global: + mon pg warn min per osd: 2 + osd: + osd pool default size: 2 + osd objectstore: filestore + osd sloppy crc: true + client: + rbd default features: 5 +roles: +- - mon.a + - mds.a + - osd.0 + - osd.1 + - osd.2 + - mgr.x +- - mon.b + - mgr.y +- - mon.c + - osd.3 + - osd.4 + - osd.5 +- - osd.6 + - osd.7 + - osd.8 + - client.0 +tasks: +- ssh-keys: +- ceph-deploy: + branch: + stable: kraken + skip-mgr: True +- ceph-deploy.upgrade: + branch: + dev: luminous + setup-mgr-node: True + check-for-healthy: True + roles: + - mon.a + - mon.b + - mon.c +- workunit: + clients: + all: + - kernel_untar_build.sh +- systemd: +- workunit: + clients: + all: + - rados/load-gen-mix.sh diff --git a/ceph/qa/suites/upgrade/kraken-x/parallel/0-cluster/start.yaml b/ceph/qa/suites/upgrade/kraken-x/parallel/0-cluster/start.yaml index 0dc9dd2bc..f5a883a39 100644 --- a/ceph/qa/suites/upgrade/kraken-x/parallel/0-cluster/start.yaml +++ b/ceph/qa/suites/upgrade/kraken-x/parallel/0-cluster/start.yaml @@ -18,6 +18,7 @@ roles: - client.1 - client.2 - client.3 +- - client.4 overrides: ceph: log-whitelist: diff --git a/ceph/qa/suites/upgrade/kraken-x/parallel/4-luminous.yaml b/ceph/qa/suites/upgrade/kraken-x/parallel/4-luminous.yaml new file mode 100644 index 000000000..80c2b9dbd --- /dev/null +++ b/ceph/qa/suites/upgrade/kraken-x/parallel/4-luminous.yaml @@ -0,0 +1,4 @@ +tasks: +- exec: + osd.0: + - ceph osd require-osd-release luminous diff --git a/ceph/qa/suites/upgrade/kraken-x/parallel/5-workload.yaml b/ceph/qa/suites/upgrade/kraken-x/parallel/5-workload.yaml new file mode 100644 index 000000000..851c5c8cb --- /dev/null +++ b/ceph/qa/suites/upgrade/kraken-x/parallel/5-workload.yaml @@ -0,0 +1,11 @@ +meta: +- desc: | + run basic import/export cli tests for rbd on not upgrated client.4 + (covers issue http://tracker.ceph.com/issues/21660) +tasks: + - workunit: + branch: kraken + clients: + client.4: + - rbd/import_export.sh + - print: "**** done rbd/import_export.sh 5-workload" diff --git a/ceph/qa/suites/upgrade/kraken-x/parallel/6-luminous-with-mgr.yaml b/ceph/qa/suites/upgrade/kraken-x/parallel/6-luminous-with-mgr.yaml new file mode 120000 index 000000000..5c72153e1 --- /dev/null +++ b/ceph/qa/suites/upgrade/kraken-x/parallel/6-luminous-with-mgr.yaml @@ -0,0 +1 @@ +../../../../releases/luminous-with-mgr.yaml \ No newline at end of file diff --git a/ceph/qa/suites/upgrade/kraken-x/parallel/7-final-workload/+ b/ceph/qa/suites/upgrade/kraken-x/parallel/7-final-workload/+ new file mode 100644 index 000000000..e69de29bb diff --git a/ceph/qa/suites/upgrade/jewel-x/parallel/5-final-workload/blogbench.yaml b/ceph/qa/suites/upgrade/kraken-x/parallel/7-final-workload/blogbench.yaml similarity index 100% rename from ceph/qa/suites/upgrade/jewel-x/parallel/5-final-workload/blogbench.yaml rename to ceph/qa/suites/upgrade/kraken-x/parallel/7-final-workload/blogbench.yaml diff --git a/ceph/qa/suites/upgrade/jewel-x/parallel/5-final-workload/rados-snaps-few-objects.yaml b/ceph/qa/suites/upgrade/kraken-x/parallel/7-final-workload/rados-snaps-few-objects.yaml similarity index 100% rename from ceph/qa/suites/upgrade/jewel-x/parallel/5-final-workload/rados-snaps-few-objects.yaml rename to ceph/qa/suites/upgrade/kraken-x/parallel/7-final-workload/rados-snaps-few-objects.yaml diff --git a/ceph/qa/suites/upgrade/jewel-x/parallel/5-final-workload/rados_loadgenmix.yaml b/ceph/qa/suites/upgrade/kraken-x/parallel/7-final-workload/rados_loadgenmix.yaml similarity index 100% rename from ceph/qa/suites/upgrade/jewel-x/parallel/5-final-workload/rados_loadgenmix.yaml rename to ceph/qa/suites/upgrade/kraken-x/parallel/7-final-workload/rados_loadgenmix.yaml diff --git a/ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/rados_mon_thrash.yaml b/ceph/qa/suites/upgrade/kraken-x/parallel/7-final-workload/rados_mon_thrash.yaml similarity index 100% rename from ceph/qa/suites/upgrade/kraken-x/parallel/5-final-workload/rados_mon_thrash.yaml rename to ceph/qa/suites/upgrade/kraken-x/parallel/7-final-workload/rados_mon_thrash.yaml diff --git a/ceph/qa/suites/upgrade/jewel-x/parallel/5-final-workload/rbd_cls.yaml b/ceph/qa/suites/upgrade/kraken-x/parallel/7-final-workload/rbd_cls.yaml similarity index 100% rename from ceph/qa/suites/upgrade/jewel-x/parallel/5-final-workload/rbd_cls.yaml rename to ceph/qa/suites/upgrade/kraken-x/parallel/7-final-workload/rbd_cls.yaml diff --git a/ceph/qa/suites/upgrade/jewel-x/parallel/5-final-workload/rbd_import_export.yaml b/ceph/qa/suites/upgrade/kraken-x/parallel/7-final-workload/rbd_import_export.yaml similarity index 100% rename from ceph/qa/suites/upgrade/jewel-x/parallel/5-final-workload/rbd_import_export.yaml rename to ceph/qa/suites/upgrade/kraken-x/parallel/7-final-workload/rbd_import_export.yaml diff --git a/ceph/qa/suites/upgrade/jewel-x/parallel/5-final-workload/rgw_swift.yaml b/ceph/qa/suites/upgrade/kraken-x/parallel/7-final-workload/rgw_swift.yaml similarity index 100% rename from ceph/qa/suites/upgrade/jewel-x/parallel/5-final-workload/rgw_swift.yaml rename to ceph/qa/suites/upgrade/kraken-x/parallel/7-final-workload/rgw_swift.yaml diff --git a/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/start.yaml b/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/start.yaml index 1e8d5a58d..3684b1e0a 100644 --- a/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/start.yaml +++ b/ceph/qa/suites/upgrade/luminous-x/parallel/0-cluster/start.yaml @@ -18,6 +18,7 @@ roles: - client.1 - client.2 - client.3 +- - client.4 overrides: ceph: log-whitelist: diff --git a/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_import_export_no_upgrated.yaml b/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_import_export_no_upgrated.yaml new file mode 100644 index 000000000..5de8a2361 --- /dev/null +++ b/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_import_export_no_upgrated.yaml @@ -0,0 +1,13 @@ +meta: +- desc: | + run basic import/export cli tests for rbd + on NO upgrated client +tasks: + - workunit: + branch: luminous + clients: + client.4: + - rbd/import_export.sh + env: + RBD_CREATE_ARGS: --new-format + - print: "**** done rbd/import_export.sh 4-final-workload on NO upgrated client" diff --git a/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_import_export.yaml b/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_import_export_upgrated.yaml similarity index 65% rename from ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_import_export.yaml rename to ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_import_export_upgrated.yaml index 46e135506..2c7c484e1 100644 --- a/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_import_export.yaml +++ b/ceph/qa/suites/upgrade/luminous-x/parallel/5-final-workload/rbd_import_export_upgrated.yaml @@ -1,6 +1,7 @@ meta: - desc: | run basic import/export cli tests for rbd + on upgrated client tasks: - workunit: clients: @@ -8,4 +9,4 @@ tasks: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - - print: "**** done rbd/import_export.sh 4-final-workload" + - print: "**** done rbd/import_export.sh 4-final-workload on upgrated client" diff --git a/ceph/qa/suites/upgrade/luminous-x/stress-split-erasure-code/1-ceph-install b/ceph/qa/suites/upgrade/luminous-x/stress-split-erasure-code/1-ceph-install new file mode 120000 index 000000000..0479ac542 --- /dev/null +++ b/ceph/qa/suites/upgrade/luminous-x/stress-split-erasure-code/1-ceph-install @@ -0,0 +1 @@ +../stress-split/1-ceph-install/ \ No newline at end of file diff --git a/ceph/qa/tasks/ceph.py b/ceph/qa/tasks/ceph.py index a37fec1c0..72f265375 100644 --- a/ceph/qa/tasks/ceph.py +++ b/ceph/qa/tasks/ceph.py @@ -43,7 +43,9 @@ def generate_caps(type_): osd='allow *', ), mgr=dict( - mon='allow *', + mon='allow profile mgr', + osd='allow *', + mds='allow *', ), mds=dict( mon='allow *', @@ -338,17 +340,18 @@ def create_rbd_pool(ctx, config): remote=mon_remote, ceph_cluster=cluster_name, ) - log.info('Creating RBD pool') - mon_remote.run( - args=['sudo', 'ceph', '--cluster', cluster_name, - 'osd', 'pool', 'create', 'rbd', '8']) - mon_remote.run( - args=[ - 'sudo', 'ceph', '--cluster', cluster_name, - 'osd', 'pool', 'application', 'enable', - 'rbd', 'rbd', '--yes-i-really-mean-it' - ], - check_status=False) + if config.get('create_rbd_pool', True): + log.info('Creating RBD pool') + mon_remote.run( + args=['sudo', 'ceph', '--cluster', cluster_name, + 'osd', 'pool', 'create', 'rbd', '8']) + mon_remote.run( + args=[ + 'sudo', 'ceph', '--cluster', cluster_name, + 'osd', 'pool', 'application', 'enable', + 'rbd', 'rbd', '--yes-i-really-mean-it' + ], + check_status=False) yield @contextlib.contextmanager @@ -365,7 +368,8 @@ def cephfs_setup(ctx, config): if mdss.remotes: log.info('Setting up CephFS filesystem...') - fs = Filesystem(ctx, name='cephfs', create=True) + fs = Filesystem(ctx, name='cephfs', create=True, + ec_profile=config.get('cephfs_ec_profile', None)) is_active_mds = lambda role: 'mds.' in role and not role.endswith('-s') and '-s-' not in role all_roles = [item for remote_roles in mdss.remotes.values() for item in remote_roles] diff --git a/ceph/qa/tasks/ceph_deploy.py b/ceph/qa/tasks/ceph_deploy.py index b22c32113..38fbe43c2 100644 --- a/ceph/qa/tasks/ceph_deploy.py +++ b/ceph/qa/tasks/ceph_deploy.py @@ -15,6 +15,7 @@ from teuthology.config import config as teuth_config from teuthology.task import install as install_fn from teuthology.orchestra import run from tasks.cephfs.filesystem import Filesystem +from teuthology.misc import wait_until_healthy log = logging.getLogger(__name__) @@ -27,7 +28,8 @@ def download_ceph_deploy(ctx, config): will use that instead. The `bootstrap` script is ran, with the argument obtained from `python_version`, if specified. """ - ceph_admin = ctx.cluster.only(teuthology.get_first_mon(ctx, config)) + # use mon.a for ceph_admin + (ceph_admin,) = ctx.cluster.only('mon.a').remotes.iterkeys() try: py_ver = str(config['python_version']) @@ -41,8 +43,7 @@ def download_ceph_deploy(ctx, config): )) log.info("Installing Python") - for admin in ceph_admin.remotes: - system_type = teuthology.get_system_type(admin) + system_type = teuthology.get_system_type(ceph_admin) if system_type == 'rpm': package = 'python34' if py_ver == '3' else 'python' @@ -145,7 +146,7 @@ def get_nodes_using_role(ctx, target_role): # Prepare a modified version of cluster.remotes with ceph-deploy-ized names modified_remotes = {} - + ceph_deploy_mapped = dict() for _remote, roles_for_host in ctx.cluster.remotes.iteritems(): modified_remotes[_remote] = [] for svc_id in roles_for_host: @@ -156,13 +157,16 @@ def get_nodes_using_role(ctx, target_role): nodes_of_interest.append(fqdn) else: nodes_of_interest.append(nodename) - - modified_remotes[_remote].append( - "{0}.{1}".format(target_role, nodename)) + mapped_role = "{0}.{1}".format(target_role, nodename) + modified_remotes[_remote].append(mapped_role) + # keep dict of mapped role for later use by tasks + # eg. mon.a => mon.node1 + ceph_deploy_mapped[svc_id] = mapped_role else: modified_remotes[_remote].append(svc_id) ctx.cluster.remotes = modified_remotes + ctx.cluster.mapped_role = ceph_deploy_mapped return nodes_of_interest @@ -213,8 +217,8 @@ def build_ceph_cluster(ctx, config): # Expect to find ceph_admin on the first mon by ID, same place that the download task # puts it. Remember this here, because subsequently IDs will change from those in # the test config to those that ceph-deploy invents. - (ceph_admin,) = ctx.cluster.only( - teuthology.get_first_mon(ctx, config)).remotes.iterkeys() + + (ceph_admin,) = ctx.cluster.only('mon.a').remotes.iterkeys() def execute_ceph_deploy(cmd): """Remotely execute a ceph_deploy command""" @@ -241,10 +245,16 @@ def build_ceph_cluster(ctx, config): mds_nodes = " ".join(mds_nodes) mon_node = get_nodes_using_role(ctx, 'mon') mon_nodes = " ".join(mon_node) - mgr_nodes = get_nodes_using_role(ctx, 'mgr') - mgr_nodes = " ".join(mgr_nodes) + # skip mgr based on config item + # this is needed when test uses latest code to install old ceph + # versions + skip_mgr = config.get('skip-mgr', False) + if not skip_mgr: + mgr_nodes = get_nodes_using_role(ctx, 'mgr') + mgr_nodes = " ".join(mgr_nodes) new_mon = './ceph-deploy new' + " " + mon_nodes - mgr_create = './ceph-deploy mgr create' + " " + mgr_nodes + if not skip_mgr: + mgr_create = './ceph-deploy mgr create' + " " + mgr_nodes mon_hostname = mon_nodes.split(' ')[0] mon_hostname = str(mon_hostname) gather_keys = './ceph-deploy gatherkeys' + " " + mon_hostname @@ -307,7 +317,8 @@ def build_ceph_cluster(ctx, config): estatus_gather = execute_ceph_deploy(gather_keys) - execute_ceph_deploy(mgr_create) + if not skip_mgr: + execute_ceph_deploy(mgr_create) if mds_nodes: estatus_mds = execute_ceph_deploy(deploy_mds) @@ -334,7 +345,7 @@ def build_ceph_cluster(ctx, config): # first check for filestore, default is bluestore with ceph-deploy if config.get('filestore') is not None: osd_create_cmd += '--filestore ' - else: + elif config.get('bluestore') is not None: osd_create_cmd += '--bluestore ' if config.get('dmcrypt') is not None: osd_create_cmd += '--dmcrypt ' @@ -414,7 +425,7 @@ def build_ceph_cluster(ctx, config): if mds_nodes: log.info('Configuring CephFS...') - ceph_fs = Filesystem(ctx, create=True) + Filesystem(ctx, create=True) elif not config.get('only_mon'): raise RuntimeError( "The cluster is NOT operational due to insufficient OSDs") @@ -524,7 +535,7 @@ def cli_test(ctx, config): """Either use git path or repo path """ args = ['cd', conf_dir, run.Raw(';')] if path: - args.append('{path}/ceph-deploy/ceph-deploy'.format(path=path)); + args.append('{path}/ceph-deploy/ceph-deploy'.format(path=path)) else: args.append('ceph-deploy') args.append(run.Raw(cmd)) @@ -608,11 +619,11 @@ def cli_test(ctx, config): log.info("Waiting for cluster to become healthy") with contextutil.safe_while(sleep=10, tries=6, action='check health') as proceed: - while proceed(): - r = remote.run(args=['sudo', 'ceph', 'health'], stdout=StringIO()) - out = r.stdout.getvalue() - if (out.split(None,1)[0] == 'HEALTH_OK'): - break + while proceed(): + r = remote.run(args=['sudo', 'ceph', 'health'], stdout=StringIO()) + out = r.stdout.getvalue() + if (out.split(None, 1)[0] == 'HEALTH_OK'): + break rgw_install = 'install {branch} --rgw {node}'.format( branch=test_branch, node=nodename, @@ -679,6 +690,108 @@ def single_node_test(ctx, config): yield +@contextlib.contextmanager +def upgrade(ctx, config): + """ + Upgrade using ceph-deploy + eg: + ceph-deploy.upgrade: + # to upgrade to specific branch, use + branch: + stable: jewel + # to setup mgr node, use + setup-mgr-node: True + # to wait for cluster to be healthy after all upgrade, use + wait-for-healthy: True + role: (upgrades the below roles serially) + mon.a + mon.b + osd.0 + """ + roles = config.get('roles') + # get the roles that are mapped as per ceph-deploy + # roles are mapped for mon/mds eg: mon.a => mon.host_short_name + mapped_role = ctx.cluster.mapped_role + if config.get('branch'): + branch = config.get('branch') + (var, val) = branch.items()[0] + ceph_branch = '--{var}={val}'.format(var=var, val=val) + else: + # default to master + ceph_branch = '--dev=master' + # get the node used for initial deployment which is mon.a + mon_a = mapped_role.get('mon.a') + (ceph_admin,) = ctx.cluster.only(mon_a).remotes.iterkeys() + testdir = teuthology.get_testdir(ctx) + cmd = './ceph-deploy install ' + ceph_branch + for role in roles: + # check if this role is mapped (mon or mds) + if mapped_role.get(role): + role = mapped_role.get(role) + remotes_and_roles = ctx.cluster.only(role).remotes + for remote, roles in remotes_and_roles.iteritems(): + nodename = remote.shortname + cmd = cmd + ' ' + nodename + log.info("Upgrading ceph on %s", nodename) + ceph_admin.run( + args=[ + 'cd', + '{tdir}/ceph-deploy'.format(tdir=testdir), + run.Raw('&&'), + run.Raw(cmd), + ], + ) + # restart all ceph services, ideally upgrade should but it does not + remote.run( + args=[ + 'sudo', 'systemctl', 'restart', 'ceph.target' + ] + ) + ceph_admin.run(args=['sudo', 'ceph', '-s']) + + # workaround for http://tracker.ceph.com/issues/20950 + # write the correct mgr key to disk + if config.get('setup-mgr-node', None): + mons = ctx.cluster.only(teuthology.is_type('mon')) + for remote, roles in mons.remotes.iteritems(): + remote.run( + args=[ + run.Raw('sudo ceph auth get client.bootstrap-mgr'), + run.Raw('|'), + run.Raw('sudo tee'), + run.Raw('/var/lib/ceph/bootstrap-mgr/ceph.keyring') + ] + ) + + if config.get('setup-mgr-node', None): + mgr_nodes = get_nodes_using_role(ctx, 'mgr') + mgr_nodes = " ".join(mgr_nodes) + mgr_install = './ceph-deploy install --mgr ' + ceph_branch + " " + mgr_nodes + mgr_create = './ceph-deploy mgr create' + " " + mgr_nodes + # install mgr + ceph_admin.run( + args=[ + 'cd', + '{tdir}/ceph-deploy'.format(tdir=testdir), + run.Raw('&&'), + run.Raw(mgr_install), + ], + ) + # create mgr + ceph_admin.run( + args=[ + 'cd', + '{tdir}/ceph-deploy'.format(tdir=testdir), + run.Raw('&&'), + run.Raw(mgr_create), + ], + ) + ceph_admin.run(args=['sudo', 'ceph', '-s']) + if config.get('wait-for-healthy', None): + wait_until_healthy(ctx, ceph_admin, use_sudo=True) + yield + + @contextlib.contextmanager def task(ctx, config): """ @@ -694,12 +807,15 @@ def task(ctx, config): branch: stable: bobtail mon_initial_members: 1 + ceph-deploy-branch: my-ceph-deploy-branch only_mon: true keep_running: true # either choose bluestore or filestore, default is bluestore bluestore: True # or filestore: True + # skip install of mgr for old release using below flag + skip-mgr: True ( default is False ) tasks: - install: diff --git a/ceph/qa/tasks/ceph_manager.py b/ceph/qa/tasks/ceph_manager.py index 9da03bdd9..5a89f235f 100644 --- a/ceph/qa/tasks/ceph_manager.py +++ b/ceph/qa/tasks/ceph_manager.py @@ -111,12 +111,12 @@ class Thrasher: self.stopping = False self.logger = logger self.config = config - self.revive_timeout = self.config.get("revive_timeout", 150) + self.revive_timeout = self.config.get("revive_timeout", 360) self.pools_to_fix_pgp_num = set() if self.config.get('powercycle'): self.revive_timeout += 120 self.clean_wait = self.config.get('clean_wait', 0) - self.minin = self.config.get("min_in", 3) + self.minin = self.config.get("min_in", 4) self.chance_move_pg = self.config.get('chance_move_pg', 1.0) self.sighup_delay = self.config.get('sighup_delay') self.optrack_toggle_delay = self.config.get('optrack_toggle_delay') @@ -286,6 +286,7 @@ class Thrasher: pg=pg, id=exp_osd)) # export + # Can't use new export-remove op since this is part of upgrade testing cmd = prefix + "--op export --pgid {pg} --file {file}" cmd = cmd.format(id=exp_osd, pg=pg, file=exp_path) proc = exp_remote.run(args=cmd) @@ -294,7 +295,7 @@ class Thrasher: "export failure with status {ret}". format(ret=proc.exitstatus)) # remove - cmd = prefix + "--op remove --pgid {pg}" + cmd = prefix + "--force --op remove --pgid {pg}" cmd = cmd.format(id=exp_osd, pg=pg) proc = exp_remote.run(args=cmd) if proc.exitstatus: @@ -767,7 +768,7 @@ class Thrasher: osd_debug_skip_full_check_in_backfill_reservation to force the more complicated check in do_scan to be exercised. - Then, verify that all backfills stop. + Then, verify that all backfillings stop. """ self.log("injecting backfill full") for i in self.live_osds: @@ -779,13 +780,13 @@ class Thrasher: check_status=True, timeout=30, stdout=DEVNULL) for i in range(30): status = self.ceph_manager.compile_pg_status() - if 'backfill' not in status.keys(): + if 'backfilling' not in status.keys(): break self.log( - "waiting for {still_going} backfills".format( - still_going=status.get('backfill'))) + "waiting for {still_going} backfillings".format( + still_going=status.get('backfilling'))) time.sleep(1) - assert('backfill' not in self.ceph_manager.compile_pg_status().keys()) + assert('backfilling' not in self.ceph_manager.compile_pg_status().keys()) for i in self.live_osds: self.ceph_manager.set_config( i, @@ -2043,7 +2044,7 @@ class CephManager: for pg in pgs: if (pg['state'].count('active') and not pg['state'].count('recover') and - not pg['state'].count('backfill') and + not pg['state'].count('backfilling') and not pg['state'].count('stale')): num += 1 return num @@ -2217,6 +2218,8 @@ class CephManager: else: self.log("no progress seen, keeping timeout for now") if now - start >= timeout: + if self.is_recovered(): + break self.log('dumping pgs') out = self.raw_cluster_cmd('pg', 'dump') self.log(out) @@ -2317,6 +2320,30 @@ class CephManager: time.sleep(3) self.log("active!") + def wait_till_pg_convergence(self, timeout=None): + start = time.time() + old_stats = None + active_osds = [osd['osd'] for osd in self.get_osd_dump() + if osd['in'] and osd['up']] + while True: + # strictly speaking, no need to wait for mon. but due to the + # "ms inject socket failures" setting, the osdmap could be delayed, + # so mgr is likely to ignore the pg-stat messages with pgs serving + # newly created pools which is not yet known by mgr. so, to make sure + # the mgr is updated with the latest pg-stats, waiting for mon/mgr is + # necessary. + self.flush_pg_stats(active_osds) + new_stats = dict((stat['pgid'], stat['state']) + for stat in self.get_pg_stats()) + if old_stats == new_stats: + return old_stats + if timeout is not None: + assert time.time() - start < timeout, \ + 'failed to reach convergence before %d secs' % timeout + old_stats = new_stats + # longer than mgr_stats_period + time.sleep(5 + 1) + def mark_out_osd(self, osd): """ Wrapper to mark osd out. @@ -2368,7 +2395,7 @@ class CephManager: time.sleep(2) self.ctx.daemons.get_daemon('osd', osd, self.cluster).stop() - def revive_osd(self, osd, timeout=150, skip_admin_check=False): + def revive_osd(self, osd, timeout=360, skip_admin_check=False): """ Revive osds by either power cycling (if indicated by the config) or by restarting. diff --git a/ceph/qa/tasks/ceph_objectstore_tool.py b/ceph/qa/tasks/ceph_objectstore_tool.py index 3dc49624c..912577317 100644 --- a/ceph/qa/tasks/ceph_objectstore_tool.py +++ b/ceph/qa/tasks/ceph_objectstore_tool.py @@ -591,7 +591,7 @@ def test_objectstore(ctx, config, cli_remote, REP_POOL, REP_NAME, ec=False): continue for pg in pgs[osdid]: - cmd = ((prefix + "--op remove --pgid {pg}"). + cmd = ((prefix + "--force --op remove --pgid {pg}"). format(pg=pg, id=osdid)) proc = remote.run(args=cmd, check_status=False, stdout=StringIO()) diff --git a/ceph/qa/tasks/cephfs/filesystem.py b/ceph/qa/tasks/cephfs/filesystem.py index 44f6cbaf1..9638fd55c 100644 --- a/ceph/qa/tasks/cephfs/filesystem.py +++ b/ceph/qa/tasks/cephfs/filesystem.py @@ -374,10 +374,12 @@ class Filesystem(MDSCluster): This object is for driving a CephFS filesystem. The MDS daemons driven by MDSCluster may be shared with other Filesystems. """ - def __init__(self, ctx, fscid=None, name=None, create=False): + def __init__(self, ctx, fscid=None, name=None, create=False, + ec_profile=None): super(Filesystem, self).__init__(ctx) self.name = name + self.ec_profile = ec_profile self.id = None self.metadata_pool_name = None self.metadata_overlay = False @@ -473,8 +475,22 @@ class Filesystem(MDSCluster): self.name, self.metadata_pool_name, data_pool_name, '--allow-dangerous-metadata-overlay') else: - self.mon_manager.raw_cluster_cmd('osd', 'pool', 'create', - data_pool_name, pgs_per_fs_pool.__str__()) + if self.ec_profile: + log.info("EC profile is %s", self.ec_profile) + cmd = ['osd', 'erasure-code-profile', 'set', data_pool_name] + cmd.extend(self.ec_profile) + self.mon_manager.raw_cluster_cmd(*cmd) + self.mon_manager.raw_cluster_cmd( + 'osd', 'pool', 'create', + data_pool_name, pgs_per_fs_pool.__str__(), 'erasure', + data_pool_name) + self.mon_manager.raw_cluster_cmd( + 'osd', 'pool', 'set', + data_pool_name, 'allow_ec_overwrites', 'true') + else: + self.mon_manager.raw_cluster_cmd( + 'osd', 'pool', 'create', + data_pool_name, pgs_per_fs_pool.__str__()) self.mon_manager.raw_cluster_cmd('fs', 'new', self.name, self.metadata_pool_name, data_pool_name) self.check_pool_application(self.metadata_pool_name) diff --git a/ceph/qa/tasks/cephfs/test_client_limits.py b/ceph/qa/tasks/cephfs/test_client_limits.py index b06d2a1d2..cb5e3a462 100644 --- a/ceph/qa/tasks/cephfs/test_client_limits.py +++ b/ceph/qa/tasks/cephfs/test_client_limits.py @@ -29,7 +29,7 @@ class TestClientLimits(CephFSTestCase): REQUIRE_KCLIENT_REMOTE = True CLIENTS_REQUIRED = 2 - def _test_client_pin(self, use_subdir): + def _test_client_pin(self, use_subdir, open_files): """ When a client pins an inode in its cache, for example because the file is held open, it should reject requests from the MDS to trim these caps. The MDS should complain @@ -39,13 +39,16 @@ class TestClientLimits(CephFSTestCase): :param use_subdir: whether to put test files in a subdir or use root """ - cache_size = 100 - open_files = 200 + cache_size = open_files/2 self.set_conf('mds', 'mds cache size', cache_size) self.fs.mds_fail_restart() self.fs.wait_for_daemons() + mds_min_caps_per_client = int(self.fs.get_config("mds_min_caps_per_client")) + self.assertTrue(open_files >= mds_min_caps_per_client) + mds_max_ratio_caps_per_client = float(self.fs.get_config("mds_max_ratio_caps_per_client")) + mount_a_client_id = self.mount_a.get_global_id() path = "subdir/mount_a" if use_subdir else "mount_a" open_proc = self.mount_a.open_n_background(path, open_files) @@ -62,8 +65,7 @@ class TestClientLimits(CephFSTestCase): # MDS should not be happy about that, as the client is failing to comply # with the SESSION_RECALL messages it is being sent mds_recall_state_timeout = float(self.fs.get_config("mds_recall_state_timeout")) - self.wait_for_health("MDS_CLIENT_RECALL", - mds_recall_state_timeout + 10) + self.wait_for_health("MDS_CLIENT_RECALL", mds_recall_state_timeout+10) # We can also test that the MDS health warning for oversized # cache is functioning as intended. @@ -82,19 +84,31 @@ class TestClientLimits(CephFSTestCase): # The remaining caps should comply with the numbers sent from MDS in SESSION_RECALL message, # which depend on the caps outstanding, cache size and overall ratio - self.wait_until_equal( - lambda: self.get_session(mount_a_client_id)['num_caps'], - int(open_files * 0.2), - timeout=30, - reject_fn=lambda x: x < int(open_files*0.2)) + recall_expected_value = int((1.0-mds_max_ratio_caps_per_client)*(open_files+2)) + def expected_caps(): + num_caps = self.get_session(mount_a_client_id)['num_caps'] + if num_caps < mds_min_caps_per_client: + raise RuntimeError("client caps fell below min!") + elif num_caps == mds_min_caps_per_client: + return True + elif recall_expected_value*.95 <= num_caps <= recall_expected_value*1.05: + return True + else: + return False + + self.wait_until_true(expected_caps, timeout=60) @needs_trimming def test_client_pin_root(self): - self._test_client_pin(False) + self._test_client_pin(False, 400) @needs_trimming def test_client_pin(self): - self._test_client_pin(True) + self._test_client_pin(True, 800) + + @needs_trimming + def test_client_pin_mincaps(self): + self._test_client_pin(True, 200) def test_client_release_bug(self): """ diff --git a/ceph/qa/tasks/cephfs/test_volume_client.py b/ceph/qa/tasks/cephfs/test_volume_client.py index 65dc9a9eb..0876af96e 100644 --- a/ceph/qa/tasks/cephfs/test_volume_client.py +++ b/ceph/qa/tasks/cephfs/test_volume_client.py @@ -355,11 +355,11 @@ vc.disconnect() :return: """ - # Because the teuthology config template sets mon_pg_warn_max_per_osd to + # Because the teuthology config template sets mon_max_pg_per_osd to # 10000 (i.e. it just tries to ignore health warnings), reset it to something # sane before using volume_client, to avoid creating pools with absurdly large # numbers of PGs. - self.set_conf("global", "mon pg warn max per osd", "300") + self.set_conf("global", "mon max pg per osd", "300") for mon_daemon_state in self.ctx.daemons.iter_daemons_of_role('mon'): mon_daemon_state.restart() @@ -368,7 +368,7 @@ vc.disconnect() # Calculate how many PGs we'll expect the new volume pool to have osd_map = json.loads(self.fs.mon_manager.raw_cluster_cmd('osd', 'dump', '--format=json-pretty')) - max_per_osd = int(self.fs.get_config('mon_pg_warn_max_per_osd')) + max_per_osd = int(self.fs.get_config('mon_max_pg_per_osd')) osd_count = len(osd_map['osds']) max_overall = osd_count * max_per_osd @@ -764,7 +764,7 @@ vc.disconnect() # auth ID belongs to, the auth ID's authorized access levels # for different volumes, versioning details, etc. expected_auth_metadata = { - u"version": 1, + u"version": 2, u"compat_version": 1, u"dirty": False, u"tenant_id": u"tenant1", @@ -791,7 +791,7 @@ vc.disconnect() # Verify that the volume metadata file stores info about auth IDs # and their access levels to the volume, versioning details, etc. expected_vol_metadata = { - u"version": 1, + u"version": 2, u"compat_version": 1, u"auths": { u"guest": { @@ -905,3 +905,112 @@ vc.disconnect() volume_id=volume_id, auth_id=guestclient["auth_id"], ))) + + def test_put_object(self): + vc_mount = self.mounts[1] + vc_mount.umount_wait() + self._configure_vc_auth(vc_mount, "manila") + + obj_data = 'test data' + obj_name = 'test_vc_obj_1' + pool_name = self.fs.get_data_pool_names()[0] + + self._volume_client_python(vc_mount, dedent(""" + vc.put_object("{pool_name}", "{obj_name}", b"{obj_data}") + """.format( + pool_name = pool_name, + obj_name = obj_name, + obj_data = obj_data + ))) + + read_data = self.fs.rados(['get', obj_name, '-'], pool=pool_name) + self.assertEqual(obj_data, read_data) + + def test_get_object(self): + vc_mount = self.mounts[1] + vc_mount.umount_wait() + self._configure_vc_auth(vc_mount, "manila") + + obj_data = 'test_data' + obj_name = 'test_vc_ob_2' + pool_name = self.fs.get_data_pool_names()[0] + + self.fs.rados(['put', obj_name, '-'], pool=pool_name, stdin_data=obj_data) + + self._volume_client_python(vc_mount, dedent(""" + data_read = vc.get_object("{pool_name}", "{obj_name}") + assert data_read == b"{obj_data}" + """.format( + pool_name = pool_name, + obj_name = obj_name, + obj_data = obj_data + ))) + + def test_delete_object(self): + vc_mount = self.mounts[1] + vc_mount.umount_wait() + self._configure_vc_auth(vc_mount, "manila") + + obj_data = 'test data' + obj_name = 'test_vc_obj_3' + pool_name = self.fs.get_data_pool_names()[0] + + self.fs.rados(['put', obj_name, '-'], pool=pool_name, stdin_data=obj_data) + + self._volume_client_python(vc_mount, dedent(""" + data_read = vc.delete_object("{pool_name}", "{obj_name}") + """.format( + pool_name = pool_name, + obj_name = obj_name, + ))) + + with self.assertRaises(CommandFailedError): + self.fs.rados(['stat', obj_name], pool=pool_name) + + # Check idempotency -- no error raised trying to delete non-existent + # object + self._volume_client_python(vc_mount, dedent(""" + data_read = vc.delete_object("{pool_name}", "{obj_name}") + """.format( + pool_name = pool_name, + obj_name = obj_name, + ))) + + def test_21501(self): + """ + Reproducer for #21501 "ceph_volume_client: sets invalid caps for + existing IDs with no caps" (http://tracker.ceph.com/issues/21501) + """ + + vc_mount = self.mounts[1] + vc_mount.umount_wait() + + # Configure vc_mount as the handle for driving volumeclient + self._configure_vc_auth(vc_mount, "manila") + + # Create a volume + group_id = "grpid" + volume_id = "volid" + mount_path = self._volume_client_python(vc_mount, dedent(""" + vp = VolumePath("{group_id}", "{volume_id}") + create_result = vc.create_volume(vp, 1024*1024*10) + print create_result['mount_path'] + """.format( + group_id=group_id, + volume_id=volume_id + ))) + + # Create an auth ID with no caps + guest_id = '21501' + self.fs.mon_manager.raw_cluster_cmd_result( + 'auth', 'get-or-create', 'client.{0}'.format(guest_id)) + + guest_mount = self.mounts[2] + guest_mount.umount_wait() + + # Set auth caps for the auth ID using the volumeclient + self._configure_guest_auth(vc_mount, guest_mount, guest_id, mount_path) + + # Mount the volume in the guest using the auth ID to assert that the + # auth caps are valid + guest_mount.mount(mount_path=mount_path) diff --git a/ceph/qa/tasks/divergent_priors2.py b/ceph/qa/tasks/divergent_priors2.py index 0e645c7c4..0ed753278 100644 --- a/ceph/qa/tasks/divergent_priors2.py +++ b/ceph/qa/tasks/divergent_priors2.py @@ -156,13 +156,7 @@ def task(ctx, config): format(fpath=FSPATH, jpath=JPATH)) pid = os.getpid() expfile = os.path.join(testdir, "exp.{pid}.out".format(pid=pid)) - cmd = ((prefix + "--op export --pgid 2.0 --file {file}"). - format(id=divergent, file=expfile)) - proc = exp_remote.run(args=cmd, wait=True, - check_status=False, stdout=StringIO()) - assert proc.exitstatus == 0 - - cmd = ((prefix + "--op remove --pgid 2.0"). + cmd = ((prefix + "--op export-remove --pgid 2.0 --file {file}"). format(id=divergent, file=expfile)) proc = exp_remote.run(args=cmd, wait=True, check_status=False, stdout=StringIO()) diff --git a/ceph/qa/tasks/mgr/mgr_test_case.py b/ceph/qa/tasks/mgr/mgr_test_case.py index a5531d33e..ec3f98d28 100644 --- a/ceph/qa/tasks/mgr/mgr_test_case.py +++ b/ceph/qa/tasks/mgr/mgr_test_case.py @@ -1,14 +1,18 @@ from unittest import case import json +import logging from teuthology import misc from tasks.ceph_test_case import CephTestCase -# TODO move definition of CephCluster +# TODO move definition of CephCluster away from the CephFS stuff from tasks.cephfs.filesystem import CephCluster +log = logging.getLogger(__name__) + + class MgrCluster(CephCluster): def __init__(self, ctx): super(MgrCluster, self).__init__(ctx) @@ -43,6 +47,12 @@ class MgrCluster(CephCluster): def get_standby_ids(self): return [s['name'] for s in self.get_mgr_map()["standbys"]] + def set_module_localized_conf(self, module, mgr_id, key, val): + self.mon_manager.raw_cluster_cmd("config-key", "set", + "mgr/{0}/{1}/{2}".format( + module, mgr_id, key + ), val) + class MgrTestCase(CephTestCase): MGRS_REQUIRED = 1 @@ -77,3 +87,84 @@ class MgrTestCase(CephTestCase): self.wait_until_true( lambda: set(self.mgr_cluster.get_standby_ids()) == expect_standbys, timeout=20) + + def _load_module(self, module_name): + loaded = json.loads(self.mgr_cluster.mon_manager.raw_cluster_cmd( + "mgr", "module", "ls"))['enabled_modules'] + if module_name in loaded: + # The enable command is idempotent, but our wait for a restart + # isn't, so let's return now if it's already loaded + return + + initial_gid = self.mgr_cluster.get_mgr_map()['active_gid'] + self.mgr_cluster.mon_manager.raw_cluster_cmd("mgr", "module", "enable", + module_name) + + # Wait for the module to load + def has_restarted(): + mgr_map = self.mgr_cluster.get_mgr_map() + done = mgr_map['active_gid'] != initial_gid and mgr_map['available'] + if done: + log.info("Restarted after module load (new active {0}/{1})".format( + mgr_map['active_name'] , mgr_map['active_gid'])) + return done + self.wait_until_true(has_restarted, timeout=30) + + + def _get_uri(self, service_name): + # Little dict hack so that I can assign into this from + # the get_or_none function + mgr_map = {'x': None} + + def _get_or_none(): + mgr_map['x'] = self.mgr_cluster.get_mgr_map() + result = mgr_map['x']['services'].get(service_name, None) + return result + + self.wait_until_true(lambda: _get_or_none() is not None, 30) + + uri = mgr_map['x']['services'][service_name] + + log.info("Found {0} at {1} (daemon {2}/{3})".format( + service_name, uri, mgr_map['x']['active_name'], + mgr_map['x']['active_gid'])) + + return uri + + + def _assign_ports(self, module_name, config_name, min_port=7789): + """ + To avoid the need to run lots of hosts in teuthology tests to + get different URLs per mgr, we will hand out different ports + to each mgr here. + + This is already taken care of for us when running in a vstart + environment. + """ + # Start handing out ports well above Ceph's range. + assign_port = min_port + + for mgr_id in self.mgr_cluster.mgr_ids: + self.mgr_cluster.mgr_stop(mgr_id) + self.mgr_cluster.mgr_fail(mgr_id) + + for mgr_id in self.mgr_cluster.mgr_ids: + log.info("Using port {0} for {1} on mgr.{2}".format( + assign_port, module_name, mgr_id + )) + self.mgr_cluster.set_module_localized_conf(module_name, mgr_id, + config_name, + str(assign_port)) + assign_port += 1 + + for mgr_id in self.mgr_cluster.mgr_ids: + self.mgr_cluster.mgr_restart(mgr_id) + + def is_available(): + mgr_map = self.mgr_cluster.get_mgr_map() + done = mgr_map['available'] + if done: + log.info("Available after assign ports (new active {0}/{1})".format( + mgr_map['active_name'] , mgr_map['active_gid'])) + return done + self.wait_until_true(is_available, timeout=30) diff --git a/ceph/qa/tasks/mgr/test_dashboard.py b/ceph/qa/tasks/mgr/test_dashboard.py new file mode 100644 index 000000000..3b8a2cc80 --- /dev/null +++ b/ceph/qa/tasks/mgr/test_dashboard.py @@ -0,0 +1,70 @@ + + +from mgr_test_case import MgrTestCase + +import logging +import requests + + +log = logging.getLogger(__name__) + + +class TestDashboard(MgrTestCase): + MGRS_REQUIRED = 3 + + def test_standby(self): + self._assign_ports("dashboard", "server_port") + self._load_module("dashboard") + + original_active = self.mgr_cluster.get_active_id() + + original_uri = self._get_uri("dashboard") + log.info("Originally running at {0}".format(original_uri)) + + self.mgr_cluster.mgr_fail(original_active) + + failed_over_uri = self._get_uri("dashboard") + log.info("After failover running at {0}".format(original_uri)) + + self.assertNotEqual(original_uri, failed_over_uri) + + # The original active daemon should have come back up as a standby + # and be doing redirects to the new active daemon + r = requests.get(original_uri, allow_redirects=False) + self.assertEqual(r.status_code, 303) + self.assertEqual(r.headers['Location'], failed_over_uri) + + def test_urls(self): + self._assign_ports("dashboard", "server_port") + self._load_module("dashboard") + + base_uri = self._get_uri("dashboard") + + # This is a very simple smoke test to check that the dashboard can + # give us a 200 response to requests. We're not testing that + # the content is correct or even renders! + + urls = [ + "/health", + "/servers", + "/osd/", + "/osd/perf/0", + "/rbd_mirroring", + "/rbd_iscsi" + ] + + failures = [] + + for url in urls: + r = requests.get(base_uri + url, allow_redirects=False) + if r.status_code >= 300 and r.status_code < 400: + log.error("Unexpected redirect to: {0} (from {1})".format( + r.headers['Location'], base_uri)) + if r.status_code != 200: + failures.append(url) + + log.info("{0}: {1} ({2} bytes)".format( + url, r.status_code, len(r.content) + )) + + self.assertListEqual(failures, []) diff --git a/ceph/qa/tasks/mgr/test_module_selftest.py b/ceph/qa/tasks/mgr/test_module_selftest.py new file mode 100644 index 000000000..2776fb872 --- /dev/null +++ b/ceph/qa/tasks/mgr/test_module_selftest.py @@ -0,0 +1,74 @@ + +import time +import requests + +from tasks.mgr.mgr_test_case import MgrTestCase + + +class TestModuleSelftest(MgrTestCase): + """ + That modules with a self-test command can be loaded and execute it + without errors. + + This is not a substitute for really testing the modules, but it + is quick and is designed to catch regressions that could occur + if data structures change in a way that breaks how the modules + touch them. + """ + MGRS_REQUIRED = 1 + + def _selftest_plugin(self, module_name): + self._load_module(module_name) + + # Execute the module's self-test routine + self.mgr_cluster.mon_manager.raw_cluster_cmd(module_name, "self-test") + + def test_zabbix(self): + self._selftest_plugin("zabbix") + + def test_prometheus(self): + self._selftest_plugin("prometheus") + + def test_influx(self): + self._selftest_plugin("influx") + + def test_selftest_run(self): + self._load_module("selftest") + self.mgr_cluster.mon_manager.raw_cluster_cmd("mgr", "self-test", "run") + + def test_selftest_command_spam(self): + # Use the selftest module to stress the mgr daemon + self._load_module("selftest") + + # Use the dashboard to test that the mgr is still able to do its job + self._assign_ports("dashboard", "server_port") + self._load_module("dashboard") + + original_active = self.mgr_cluster.get_active_id() + original_standbys = self.mgr_cluster.get_standby_ids() + + self.mgr_cluster.mon_manager.raw_cluster_cmd("mgr", "self-test", + "background", "start", + "command_spam") + + dashboard_uri = self._get_uri("dashboard") + + delay = 10 + periods = 10 + for i in range(0, periods): + t1 = time.time() + # Check that an HTTP module remains responsive + r = requests.get(dashboard_uri) + self.assertEqual(r.status_code, 200) + + # Check that a native non-module command remains responsive + self.mgr_cluster.mon_manager.raw_cluster_cmd("osd", "df") + + time.sleep(delay - (time.time() - t1)) + + self.mgr_cluster.mon_manager.raw_cluster_cmd("mgr", "self-test", + "background", "stop") + + # Check that all mgr daemons are still running + self.assertEqual(original_active, self.mgr_cluster.get_active_id()) + self.assertEqual(original_standbys, self.mgr_cluster.get_standby_ids()) diff --git a/ceph/qa/tasks/osd_max_pg_per_osd.py b/ceph/qa/tasks/osd_max_pg_per_osd.py new file mode 100644 index 000000000..b4e2aa4de --- /dev/null +++ b/ceph/qa/tasks/osd_max_pg_per_osd.py @@ -0,0 +1,126 @@ +import logging +import random + + +log = logging.getLogger(__name__) + + +def pg_num_in_all_states(pgs, *states): + return sum(1 for state in pgs.itervalues() + if all(s in state for s in states)) + + +def pg_num_in_any_state(pgs, *states): + return sum(1 for state in pgs.itervalues() + if any(s in state for s in states)) + + +def test_create_from_mon(ctx, config): + """ + osd should stop creating new pools if the number of pg it servers + exceeds the max-pg-per-osd setting, and it should resume the previously + suspended pg creations once the its pg number drops down below the setting + How it works:: + 1. set the hard limit of pg-per-osd to "2" + 2. create pool.a with pg_num=2 + # all pgs should be active+clean + 2. create pool.b with pg_num=2 + # new pgs belonging to this pool should be unknown (the primary osd + reaches the limit) or creating (replica osd reaches the limit) + 3. remove pool.a + 4. all pg belonging to pool.b should be active+clean + """ + pg_num = config.get('pg_num', 2) + manager = ctx.managers['ceph'] + log.info('1. creating pool.a') + pool_a = manager.create_pool_with_unique_name(pg_num) + manager.wait_for_clean() + assert manager.get_num_active_clean() == pg_num + + log.info('2. creating pool.b') + pool_b = manager.create_pool_with_unique_name(pg_num) + pg_states = manager.wait_till_pg_convergence(300) + pg_created = pg_num_in_all_states(pg_states, 'active', 'clean') + assert pg_created == pg_num + pg_pending = pg_num_in_any_state(pg_states, 'unknown', 'creating') + assert pg_pending == pg_num + + log.info('3. removing pool.a') + manager.remove_pool(pool_a) + pg_states = manager.wait_till_pg_convergence(300) + assert len(pg_states) == pg_num + pg_created = pg_num_in_all_states(pg_states, 'active', 'clean') + assert pg_created == pg_num + + # cleanup + manager.remove_pool(pool_b) + + +def test_create_from_peer(ctx, config): + """ + osd should stop creating new pools if the number of pg it servers + exceeds the max-pg-per-osd setting, and it should resume the previously + suspended pg creations once the its pg number drops down below the setting + + How it works:: + 0. create 4 OSDs. + 1. create pool.a with pg_num=1, size=2 + pg will be mapped to osd.0, and osd.1, and it should be active+clean + 2. create pool.b with pg_num=1, size=2. + if the pgs stuck in creating, delete the pool since the pool and try + again, eventually we'll get the pool to land on the other 2 osds that + aren't occupied by pool.a. (this will also verify that pgs for deleted + pools get cleaned out of the creating wait list.) + 3. mark an osd out. verify that some pgs get stuck stale or peering. + 4. delete a pool, verify pgs go active. + """ + pg_num = config.get('pg_num', 1) + pool_size = config.get('pool_size', 2) + from_primary = config.get('from_primary', True) + + manager = ctx.managers['ceph'] + log.info('1. creating pool.a') + pool_a = manager.create_pool_with_unique_name(pg_num) + manager.wait_for_clean() + assert manager.get_num_active_clean() == pg_num + + log.info('2. creating pool.b') + while True: + pool_b = manager.create_pool_with_unique_name(pg_num) + pg_states = manager.wait_till_pg_convergence(300) + pg_created = pg_num_in_all_states(pg_states, 'active', 'clean') + assert pg_created >= pg_num + pg_pending = pg_num_in_any_state(pg_states, 'unknown', 'creating') + assert pg_pending == pg_num * 2 - pg_created + if pg_created == pg_num * 2: + break + manager.remove_pool(pool_b) + + log.info('3. mark an osd out') + pg_stats = manager.get_pg_stats() + pg = random.choice(pg_stats) + if from_primary: + victim = pg['acting'][-1] + else: + victim = pg['acting'][0] + manager.mark_out_osd(victim) + pg_states = manager.wait_till_pg_convergence(300) + pg_stuck = pg_num_in_any_state(pg_states, 'activating', 'stale', 'peering') + assert pg_stuck > 0 + + log.info('4. removing pool.b') + manager.remove_pool(pool_b) + manager.wait_for_clean(30) + + # cleanup + manager.remove_pool(pool_a) + + +def task(ctx, config): + assert isinstance(config, dict), \ + 'osd_max_pg_per_osd task only accepts a dict for config' + manager = ctx.managers['ceph'] + if config.get('test_create_from_mon', True): + test_create_from_mon(ctx, config) + else: + test_create_from_peer(ctx, config) diff --git a/ceph/qa/tasks/reg11184.py b/ceph/qa/tasks/reg11184.py index 50e3a8b33..f24862384 100644 --- a/ceph/qa/tasks/reg11184.py +++ b/ceph/qa/tasks/reg11184.py @@ -174,19 +174,12 @@ def task(ctx, config): format(fpath=FSPATH, jpath=JPATH)) pid = os.getpid() expfile = os.path.join(testdir, "exp.{pid}.out".format(pid=pid)) - cmd = ((prefix + "--op export --pgid 2.0 --file {file}"). + cmd = ((prefix + "--op export-remove --pgid 2.0 --file {file}"). format(id=divergent, file=expfile)) proc = exp_remote.run(args=cmd, wait=True, check_status=False, stdout=StringIO()) assert proc.exitstatus == 0 - # Remove the same pg that was exported - cmd = ((prefix + "--op remove --pgid 2.0"). - format(id=divergent)) - proc = exp_remote.run(args=cmd, wait=True, - check_status=False, stdout=StringIO()) - assert proc.exitstatus == 0 - # Kill one of non-divergent OSDs log.info('killing osd.%d' % non_divergent[0]) manager.kill_osd(non_divergent[0]) @@ -194,7 +187,7 @@ def task(ctx, config): # manager.mark_out_osd(non_divergent[0]) # An empty collection for pg 2.0 might need to be cleaned up - cmd = ((prefix + "--op remove --pgid 2.0"). + cmd = ((prefix + "--force --op remove --pgid 2.0"). format(id=non_divergent[0])) proc = exp_remote.run(args=cmd, wait=True, check_status=False, stdout=StringIO()) diff --git a/ceph/qa/tasks/s3a_hadoop.py b/ceph/qa/tasks/s3a_hadoop.py index b969a36a8..c01fe1dda 100644 --- a/ceph/qa/tasks/s3a_hadoop.py +++ b/ceph/qa/tasks/s3a_hadoop.py @@ -82,7 +82,9 @@ def task(ctx, config): fix_rgw_config(rgw_node, dnsmasq_name) setup_user_bucket(rgw_node, dnsmasq_name, access_key, secret_key, bucket_name, testdir) if hadoop_ver.startswith('2.8'): - test_options = '-Dit.test=ITestS3A* -Dparallel-tests -Dscale -Dfs.s3a.scale.test.huge.filesize=128M verify' + # test all ITtests but skip AWS test using public bucket landsat-pds + # which is not available from within this test + test_options = '-Dit.test=ITestS3A* -Dit.test=\!ITestS3AAWSCredentialsProvider* -Dparallel-tests -Dscale -Dfs.s3a.scale.test.huge.filesize=128M verify' else: test_options = 'test -Dtest=S3a*,TestS3A*' try: diff --git a/ceph/qa/tasks/thrashosds.py b/ceph/qa/tasks/thrashosds.py index dbca056a0..420b73559 100644 --- a/ceph/qa/tasks/thrashosds.py +++ b/ceph/qa/tasks/thrashosds.py @@ -24,7 +24,7 @@ def task(ctx, config): cluster: (default 'ceph') the name of the cluster to thrash - min_in: (default 3) the minimum number of OSDs to keep in the + min_in: (default 4) the minimum number of OSDs to keep in the cluster min_out: (default 0) the minimum number of OSDs to keep out of the diff --git a/ceph/qa/tasks/util/rados.py b/ceph/qa/tasks/util/rados.py index 86c4b5389..a83f9e190 100644 --- a/ceph/qa/tasks/util/rados.py +++ b/ceph/qa/tasks/util/rados.py @@ -34,7 +34,7 @@ def create_ec_pool(remote, name, profile_name, pgnum, profile={}, cluster_name=" if application: remote.run(args=[ 'sudo', 'ceph', 'osd', 'pool', 'application', 'enable', name, application, '--cluster', cluster_name - ]) + ], check_status=False) # may fail as EINVAL when run in jewel upgrade test def create_replicated_pool(remote, name, pgnum, cluster_name="ceph", application=None): remote.run(args=[ @@ -43,7 +43,7 @@ def create_replicated_pool(remote, name, pgnum, cluster_name="ceph", application if application: remote.run(args=[ 'sudo', 'ceph', 'osd', 'pool', 'application', 'enable', name, application, '--cluster', cluster_name - ]) + ], check_status=False) def create_cache_pool(remote, base_name, cache_name, pgnum, size, cluster_name="ceph"): remote.run(args=[ diff --git a/ceph/qa/workunits/ceph-disk/ceph-disk-test.py b/ceph/qa/workunits/ceph-disk/ceph-disk-test.py index efc080dc0..637fa90eb 100644 --- a/ceph/qa/workunits/ceph-disk/ceph-disk-test.py +++ b/ceph/qa/workunits/ceph-disk/ceph-disk-test.py @@ -113,7 +113,7 @@ class CephDisk: LOG.debug(self.unused_disks('sd.')) if self.unused_disks('sd.'): return - modprobe = "modprobe scsi_debug vpd_use_hostno=0 add_host=1 dev_size_mb=200 ; udevadm settle" + modprobe = "modprobe scsi_debug vpd_use_hostno=0 add_host=1 dev_size_mb=300 ; udevadm settle" try: self.sh(modprobe) except: diff --git a/ceph/qa/workunits/ceph-disk/ceph-disk.sh b/ceph/qa/workunits/ceph-disk/ceph-disk.sh index 7a795b925..7102efba1 100755 --- a/ceph/qa/workunits/ceph-disk/ceph-disk.sh +++ b/ceph/qa/workunits/ceph-disk/ceph-disk.sh @@ -35,7 +35,7 @@ if ! ${PYTHON} -m pytest --version > /dev/null 2>&1; then exit 1 fi -sudo env PATH=$(dirname $0):$(dirname $0)/..:$PATH ${PYTHON} -m pytest -s -v $(dirname $0)/ceph-disk-test.py +sudo env PATH=$(dirname $0):$(dirname $0)/..:$PATH PYTHONWARNINGS=ignore ${PYTHON} -m pytest -s -v $(dirname $0)/ceph-disk-test.py result=$? sudo rm -f /lib/udev/rules.d/60-ceph-by-partuuid.rules diff --git a/ceph/qa/workunits/cephtool/test.sh b/ceph/qa/workunits/cephtool/test.sh index f5a313ea2..15344172a 100755 --- a/ceph/qa/workunits/cephtool/test.sh +++ b/ceph/qa/workunits/cephtool/test.sh @@ -1593,16 +1593,7 @@ function test_mon_osd() # When CEPH_CLI_TEST_DUP_COMMAND is set, osd create # is repeated and consumes two osd id, not just one. # - local next_osd - if test "$CEPH_CLI_TEST_DUP_COMMAND" ; then - next_osd=$((gap_start + 1)) - else - next_osd=$gap_start - fi - id=`ceph osd create` - [ "$id" = "$next_osd" ] - - next_osd=$((id + 1)) + local next_osd=$gap_start id=`ceph osd create $(uuidgen)` [ "$id" = "$next_osd" ] @@ -2162,9 +2153,12 @@ function test_mon_osd_erasure_code() ceph osd erasure-code-profile set fooprofile a=b c=d e=f --force ceph osd erasure-code-profile set fooprofile a=b c=d e=f expect_false ceph osd erasure-code-profile set fooprofile a=b c=d e=f g=h - # - # cleanup by removing profile 'fooprofile' + # ruleset-foo will work for luminous only + ceph osd erasure-code-profile set barprofile ruleset-failure-domain=host + ceph osd erasure-code-profile set barprofile crush-failure-domain=host + # clean up ceph osd erasure-code-profile rm fooprofile + ceph osd erasure-code-profile rm barprofile } function test_mon_osd_misc() diff --git a/ceph/qa/workunits/cls/test_cls_journal.sh b/ceph/qa/workunits/cls/test_cls_journal.sh new file mode 100755 index 000000000..9aa7450a9 --- /dev/null +++ b/ceph/qa/workunits/cls/test_cls_journal.sh @@ -0,0 +1,6 @@ +#!/bin/sh -e + +GTEST_FILTER=${CLS_JOURNAL_GTEST_FILTER:-*} +ceph_test_cls_journal --gtest_filter=${GTEST_FILTER} + +exit 0 diff --git a/ceph/qa/workunits/mgr/test_localpool.sh b/ceph/qa/workunits/mgr/test_localpool.sh new file mode 100755 index 000000000..c5a56a6d5 --- /dev/null +++ b/ceph/qa/workunits/mgr/test_localpool.sh @@ -0,0 +1,21 @@ +#!/bin/sh -ex + +ceph config-key set mgr/localpool/subtree host +ceph config-key set mgr/localpool/failure_domain osd +ceph mgr module enable localpool + +while ! ceph osd pool ls | grep '^by-host-' +do + sleep 5 +done + +ceph mgr module disable localpool +for p in `ceph osd pool ls | grep '^by-host-'` +do + ceph osd pool rm $p $p --yes-i-really-really-mean-it +done + +ceph config-key rm mgr/localpool/subtree +ceph config-key rm mgr/localpool/failure_domain + +echo OK diff --git a/ceph/qa/workunits/rados/test_rados_tool.sh b/ceph/qa/workunits/rados/test_rados_tool.sh index 6a3ebe0b2..87c86ee69 100755 --- a/ceph/qa/workunits/rados/test_rados_tool.sh +++ b/ceph/qa/workunits/rados/test_rados_tool.sh @@ -346,7 +346,7 @@ test_rmobj() { $CEPH_TOOL osd pool set-quota $p max_objects 1 V1=`mktemp fooattrXXXXXXX` $RADOS_TOOL put $OBJ $V1 -p $p - while ! $CEPH_TOOL osd dump | grep 'full max_objects' + while ! $CEPH_TOOL osd dump | grep 'full_no_quota max_objects' do sleep 2 done diff --git a/ceph/qa/workunits/rbd/rbd_mirror.sh b/ceph/qa/workunits/rbd/rbd_mirror.sh index 04a03a66e..5195e6cf3 100755 --- a/ceph/qa/workunits/rbd/rbd_mirror.sh +++ b/ceph/qa/workunits/rbd/rbd_mirror.sh @@ -111,6 +111,18 @@ wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' admin_daemon ${CLUSTER1} rbd mirror flush admin_daemon ${CLUSTER1} rbd mirror status +testlog "TEST: test image rename" +new_name="${image}_RENAMED" +rename_image ${CLUSTER2} ${POOL} ${image} ${new_name} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying' +admin_daemon ${CLUSTER1} rbd mirror status ${POOL}/${new_name} +admin_daemon ${CLUSTER1} rbd mirror restart ${POOL}/${new_name} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying' +rename_image ${CLUSTER2} ${POOL} ${new_name} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + testlog "TEST: failover and failback" start_mirror ${CLUSTER2} diff --git a/ceph/qa/workunits/rbd/rbd_mirror_helpers.sh b/ceph/qa/workunits/rbd/rbd_mirror_helpers.sh index 23216711e..325353b91 100755 --- a/ceph/qa/workunits/rbd/rbd_mirror_helpers.sh +++ b/ceph/qa/workunits/rbd/rbd_mirror_helpers.sh @@ -593,6 +593,16 @@ set_image_meta() rbd --cluster ${cluster} -p ${pool} image-meta set ${image} $key $val } +rename_image() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local new_name=$4 + + rbd --cluster=${cluster} -p ${pool} rename ${image} ${new_name} +} + remove_image() { local cluster=$1 diff --git a/ceph/selinux/ceph.te b/ceph/selinux/ceph.te index 552f73601..0a9349803 100644 --- a/ceph/selinux/ceph.te +++ b/ceph/selinux/ceph.te @@ -106,7 +106,7 @@ files_manage_generic_locks(ceph_t) allow ceph_t sysfs_t:dir read; allow ceph_t sysfs_t:file { read getattr open }; -allow ceph_t sysfs_t:lnk_file read; +allow ceph_t sysfs_t:lnk_file { read getattr }; allow ceph_t random_device_t:chr_file getattr; allow ceph_t urandom_device_t:chr_file getattr; diff --git a/ceph/src/.git_version b/ceph/src/.git_version index 4af702226..9b4bb5c8b 100644 --- a/ceph/src/.git_version +++ b/ceph/src/.git_version @@ -1,2 +1,2 @@ -3e7492b9ada8bdc9a5cd0feafd42fbca27f9c38e -v12.2.1 +cf0baeeeeba3b47f9427c6c97e2144b094b7e5ba +v12.2.2 diff --git a/ceph/src/90-ceph-osd.conf b/ceph/src/90-ceph-osd.conf new file mode 100644 index 000000000..c5c64bb70 --- /dev/null +++ b/ceph/src/90-ceph-osd.conf @@ -0,0 +1 @@ +fs.aio-max-nr = 1048576 diff --git a/ceph/src/CMakeLists.txt b/ceph/src/CMakeLists.txt index 3cdcb95be..3d3d2f7af 100644 --- a/ceph/src/CMakeLists.txt +++ b/ceph/src/CMakeLists.txt @@ -540,8 +540,11 @@ set(libcommon_files ${auth_files} ${mds_files}) +CHECK_C_COMPILER_FLAG("-fvar-tracking-assignments" HAS_VTA) if(HAS_VTA) - set_source_files_properties(common/config.cc + set_source_files_properties( + common/config.cc + common/options.cc PROPERTIES COMPILE_FLAGS -fno-var-tracking-assignments) endif() @@ -691,12 +694,18 @@ if (WITH_MGR) mgr/DaemonState.cc mgr/DaemonServer.cc mgr/ClusterState.cc - mgr/PyModules.cc + mgr/ActivePyModules.cc + mgr/StandbyPyModules.cc + mgr/PyModuleRegistry.cc + mgr/PyModuleRunner.cc mgr/PyFormatter.cc - mgr/PyState.cc - mgr/MgrPyModule.cc + mgr/PyOSDMap.cc + mgr/BaseMgrModule.cc + mgr/BaseMgrStandbyModule.cc + mgr/ActivePyModule.cc mgr/MgrStandby.cc mgr/Mgr.cc + mgr/Gil.cc mgr/mgr_commands.cc) add_executable(ceph-mgr ${mgr_srcs} $) @@ -726,7 +735,6 @@ add_subdirectory(ceph-volume) add_subdirectory(ceph-detect-init) ## dencoder -CHECK_C_COMPILER_FLAG("-fvar-tracking-assignments" HAS_VTA) if(HAS_VTA) set_source_files_properties(test/encoding/ceph_dencoder.cc PROPERTIES COMPILE_FLAGS -fno-var-tracking-assignments) diff --git a/ceph/src/arch/arm.c b/ceph/src/arch/arm.c index e3cfb5c59..a8562f596 100644 --- a/ceph/src/arch/arm.c +++ b/ceph/src/arch/arm.c @@ -1,3 +1,4 @@ +#include "acconfig.h" #include "arch/probe.h" /* flags we export */ @@ -45,10 +46,8 @@ int ceph_arch_arm_probe(void) ceph_arch_neon = (get_hwcap() & HWCAP_NEON) == HWCAP_NEON; #elif __aarch64__ && __linux__ ceph_arch_neon = (get_hwcap() & HWCAP_ASIMD) == HWCAP_ASIMD; -# ifdef HWCAP_CRC32 +# if defined(HAVE_ARMV8_CRC) && defined(HWCAP_CRC32) ceph_arch_aarch64_crc32 = (get_hwcap() & HWCAP_CRC32) == HWCAP_CRC32; -# else - ceph_arch_aarch64_crc32 = 0; // sorry! # endif #else if (0) diff --git a/ceph/src/ceph-disk/ceph_disk/main.py b/ceph/src/ceph-disk/ceph_disk/main.py old mode 100755 new mode 100644 index 8b0c5dbc3..6516750d2 --- a/ceph/src/ceph-disk/ceph_disk/main.py +++ b/ceph/src/ceph-disk/ceph_disk/main.py @@ -24,6 +24,7 @@ import argparse import base64 import errno import fcntl +import functools import json import logging import os @@ -41,12 +42,23 @@ import pwd import grp import textwrap import glob +import warnings CEPH_OSD_ONDISK_MAGIC = 'ceph osd volume v026' CEPH_LOCKBOX_ONDISK_MAGIC = 'ceph lockbox volume v001' KEY_MANAGEMENT_MODE_V1 = 'ceph-mon v1' +DEPRECATION_WARNING = """ +******************************************************************************* +This tool is now deprecated in favor of ceph-volume. +It is recommended to use ceph-volume for OSD deployments. For details see: + + http://docs.ceph.com/docs/master/ceph-volume/#migrating + +******************************************************************************* +""" + PTYPE = { 'regular': { 'journal': { @@ -721,6 +733,21 @@ def get_partition_mpath(dev, pnum): return None +def retry(on_error=Exception, max_tries=10, wait=0.2, backoff=0): + def wrapper(func): + @functools.wraps(func) + def repeat(*args, **kwargs): + for tries in range(max_tries - 1): + try: + return func(*args, **kwargs) + except on_error: + time.sleep(wait + backoff * tries) + return func(*args, **kwargs) + return repeat + return wrapper + + +@retry(Error) def get_partition_dev(dev, pnum): """ get the device name for a partition @@ -732,36 +759,25 @@ def get_partition_dev(dev, pnum): sda 1 -> sda1 cciss/c0d1 1 -> cciss!c0d1p1 """ - max_retry = 10 - for retry in range(0, max_retry + 1): - partname = None - error_msg = "" - if is_mpath(dev): - partname = get_partition_mpath(dev, pnum) - else: - name = get_dev_name(os.path.realpath(dev)) - sys_entry = os.path.join(BLOCKDIR, name) - error_msg = " in %s" % sys_entry - for f in os.listdir(sys_entry): - if f.startswith(name) and f.endswith(str(pnum)): - # we want the shortest name that starts with the base name - # and ends with the partition number - if not partname or len(f) < len(partname): - partname = f - if partname: - if retry: - LOG.info('Found partition %d for %s after %d tries' % - (pnum, dev, retry)) - return get_dev_path(partname) - else: - if retry < max_retry: - LOG.info('Try %d/%d : partition %d for %s does not exist%s' % - (retry + 1, max_retry, pnum, dev, error_msg)) - time.sleep(.2) - continue - else: - raise Error('partition %d for %s does not appear to exist%s' % - (pnum, dev, error_msg)) + partname = None + error_msg = "" + if is_mpath(dev): + partname = get_partition_mpath(dev, pnum) + else: + name = get_dev_name(os.path.realpath(dev)) + sys_entry = os.path.join(BLOCKDIR, name) + error_msg = " in %s" % sys_entry + for f in os.listdir(sys_entry): + if f.startswith(name) and f.endswith(str(pnum)): + # we want the shortest name that starts with the base name + # and ends with the partition number + if not partname or len(f) < len(partname): + partname = f + if partname: + return get_dev_path(partname) + else: + raise Error('partition %d for %s does not appear to exist%s' % + (pnum, dev, error_msg)) def list_all_partitions(): @@ -1374,22 +1390,14 @@ def _dmcrypt_map( raise Error('unable to map device', rawdev, e) -def dmcrypt_unmap( - _uuid -): +@retry(Error, max_tries=10, wait=0.5, backoff=1.0) +def dmcrypt_unmap(_uuid): if not os.path.exists('/dev/mapper/' + _uuid): return - retries = 0 - while True: - try: - command_check_call(['cryptsetup', 'remove', _uuid]) - break - except subprocess.CalledProcessError as e: - if retries == 10: - raise Error('unable to unmap device', _uuid, e) - else: - time.sleep(0.5 + retries * 1.0) - retries += 1 + try: + command_check_call(['cryptsetup', 'remove', _uuid]) + except subprocess.CalledProcessError as e: + raise Error('unable to unmap device', _uuid, e) def mount( @@ -1451,6 +1459,7 @@ def mount( return path +@retry(UnmountError, max_tries=3, wait=0.5, backoff=1.0) def unmount( path, do_rm=True, @@ -1458,25 +1467,17 @@ def unmount( """ Unmount and removes the given mount point. """ - retries = 0 - while True: - try: - LOG.debug('Unmounting %s', path) - command_check_call( - [ - '/bin/umount', - '--', - path, - ], - ) - break - except subprocess.CalledProcessError as e: - # on failure, retry 3 times with incremental backoff - if retries == 3: - raise UnmountError(e) - else: - time.sleep(0.5 + retries * 1.0) - retries += 1 + try: + LOG.debug('Unmounting %s', path) + command_check_call( + [ + '/bin/umount', + '--', + path, + ], + ) + except subprocess.CalledProcessError as e: + raise UnmountError(e) if not do_rm: return os.rmdir(path) @@ -1855,6 +1856,7 @@ class DevicePartition(object): return self.ptype_map[name]['ready'] @staticmethod + @retry(OSError) def factory(path, dev, args): dmcrypt_type = CryptHelpers.get_dmcrypt_type(args) if ((path is not None and is_mpath(path)) or @@ -3248,7 +3250,7 @@ def systemd_start( osd_id, ): systemd_disable(path, osd_id) - if is_mounted(path): + if os.path.ismount(path): style = ['--runtime'] else: style = [] @@ -3760,6 +3762,20 @@ def main_activate(args): ) osd_data = get_mount_point(cluster, osd_id) + args.cluster = cluster + if args.dmcrypt: + for name in Space.NAMES: + # Check if encrypted device in journal + dev_path = os.path.join(osd_data, name + '_dmcrypt') + if not os.path.exists(dev_path): + continue + partition = DevicePartition.factory( + path=None, + dev=dev_path, + args=args) + partition.rawdev = args.path + partition.map() + elif stat.S_ISDIR(mode): (cluster, osd_id) = activate_dir( path=args.path, @@ -5633,6 +5649,8 @@ def make_zap_parser(subparsers): def main(argv): + # Deprecate from the very beginning + warnings.warn(DEPRECATION_WARNING) args = parse_args(argv) setup_logging(args.verbose, args.log_stdout) @@ -5652,10 +5670,20 @@ def main(argv): CEPH_PREF_GROUP = args.setgroup if args.verbose: - args.func(args) + try: + args.func(args) + except Exception: + # warn on any exception when running with verbosity + warnings.warn(DEPRECATION_WARNING) + # but still raise the original issue + raise + else: main_catch(args.func, args) + # if there aren't any errors, still log again at the very bottom + warnings.warn(DEPRECATION_WARNING) + def setup_logging(verbose, log_stdout): loglevel = logging.WARNING @@ -5682,6 +5710,8 @@ def main_catch(func, args): func(args) except Error as e: + # warn on generic 'error' exceptions + warnings.warn(DEPRECATION_WARNING) raise SystemExit( '{prog}: {msg}'.format( prog=args.prog, @@ -5690,6 +5720,8 @@ def main_catch(func, args): ) except CephDiskException as error: + # warn on ceph-disk exceptions + warnings.warn(DEPRECATION_WARNING) exc_name = error.__class__.__name__ raise SystemExit( '{prog} {exc_name}: {msg}'.format( diff --git a/ceph/src/ceph-disk/tox.ini b/ceph/src/ceph-disk/tox.ini index a2bc483a2..bbf1e21c6 100644 --- a/ceph/src/ceph-disk/tox.ini +++ b/ceph/src/ceph-disk/tox.ini @@ -25,4 +25,4 @@ commands = coverage run --append --source=ceph_disk {envbindir}/py.test -vv {tox coverage report --show-missing [testenv:flake8] -commands = flake8 --ignore=H105,H405,E127 ceph_disk tests +commands = flake8 --ignore=H105,H405,E127,E722 ceph_disk tests diff --git a/ceph/src/ceph-volume/ceph_volume/api/__init__.py b/ceph/src/ceph-volume/ceph_volume/api/__init__.py new file mode 100644 index 000000000..ecc971299 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/api/__init__.py @@ -0,0 +1,3 @@ +""" +Device API that can be shared among other implementations. +""" diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/api.py b/ceph/src/ceph-volume/ceph_volume/api/lvm.py similarity index 89% rename from ceph/src/ceph-volume/ceph_volume/devices/lvm/api.py rename to ceph/src/ceph-volume/ceph_volume/api/lvm.py index e5bc26234..d82aee685 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/api.py +++ b/ceph/src/ceph-volume/ceph_volume/api/lvm.py @@ -131,6 +131,22 @@ def get_api_pvs(): return _output_parser(stdout, fields) +def get_lv_from_argument(argument): + """ + Helper proxy function that consumes a possible logical volume passed in from the CLI + in the form of `vg/lv`, but with some validation so that an argument that is a full + path to a device can be ignored + """ + if argument.startswith('/'): + lv = get_lv(lv_path=argument) + return lv + try: + vg_name, lv_name = argument.split('/') + except (ValueError, AttributeError): + return None + return get_lv(lv_name=lv_name, vg_name=vg_name) + + def get_lv(lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): """ Return a matching lv for the current system, requiring ``lv_name``, @@ -177,24 +193,69 @@ def create_pv(device): ]) -def create_lv(name, group, size=None, **tags): +def create_vg(name, *devices): + """ + Create a Volume Group. Command looks like:: + + vgcreate --force --yes group_name device + + Once created the volume group is returned as a ``VolumeGroup`` object + """ + process.run([ + 'sudo', + 'vgcreate', + '--force', + '--yes', + name] + list(devices) + ) + + vg = get_vg(vg_name=name) + return vg + + +def remove_lv(path): + """ + Removes a logical volume given it's absolute path. + + Will return True if the lv is successfully removed or + raises a RuntimeError if the removal fails. + """ + stdout, stderr, returncode = process.call( + [ + 'sudo', + 'lvremove', + '-v', # verbose + '-f', # force it + path + ], + show_command=True, + terminal_verbose=True, + ) + if returncode != 0: + raise RuntimeError("Unable to remove %s".format(path)) + return True + + +def create_lv(name, group, size=None, tags=None): """ Create a Logical Volume in a Volume Group. Command looks like:: lvcreate -L 50G -n gfslv vg0 - ``name``, ``group``, and ``size`` are required. Tags are optional and are "translated" to include - the prefixes for the Ceph LVM tag API. + ``name``, ``group``, are required. If ``size`` is provided it must follow + lvm's size notation (like 1G, or 20M). Tags are an optional dictionary and is expected to + conform to the convention of prefixing them with "ceph." like:: + {"ceph.block_device": "/dev/ceph/osd-1"} """ # XXX add CEPH_VOLUME_LVM_DEBUG to enable -vvvv on lv operations type_path_tag = { 'journal': 'ceph.journal_device', 'data': 'ceph.data_device', - 'block': 'ceph.block', - 'wal': 'ceph.wal', - 'db': 'ceph.db', - 'lockbox': 'ceph.lockbox_device', + 'block': 'ceph.block_device', + 'wal': 'ceph.wal_device', + 'db': 'ceph.db_device', + 'lockbox': 'ceph.lockbox_device', # XXX might not ever need this lockbox sorcery } if size: process.run([ @@ -202,7 +263,7 @@ def create_lv(name, group, size=None, **tags): 'lvcreate', '--yes', '-L', - '%sG' % size, + '%s' % size, '-n', name, group ]) # create the lv with all the space available, this is needed because the @@ -218,17 +279,15 @@ def create_lv(name, group, size=None, **tags): ]) lv = get_lv(lv_name=name, vg_name=group) - ceph_tags = {} - for k, v in tags.items(): - ceph_tags['ceph.%s' % k] = v - lv.set_tags(ceph_tags) + lv.set_tags(tags) # when creating a distinct type, the caller doesn't know what the path will # be so this function will set it after creation using the mapping - path_tag = type_path_tag[tags['type']] - lv.set_tags( - {path_tag: lv.lv_path} - ) + path_tag = type_path_tag.get(tags.get('ceph.type')) + if path_tag: + lv.set_tags( + {path_tag: lv.lv_path} + ) return lv @@ -584,6 +643,23 @@ class Volume(object): def __repr__(self): return self.__str__() + def as_dict(self): + obj = {} + obj.update(self.lv_api) + obj['tags'] = self.tags + obj['name'] = self.name + obj['type'] = self.tags['ceph.type'] + obj['path'] = self.lv_path + return obj + + def clear_tags(self): + """ + Removes all tags from the Logical Volume. + """ + for k, v in self.tags.items(): + tag = "%s=%s" % (k, v) + process.run(['sudo', 'lvchange', '--deltag', tag, self.lv_path]) + def set_tags(self, tags): """ :param tags: A dictionary of tag names and values, like:: diff --git a/ceph/src/ceph-volume/ceph_volume/decorators.py b/ceph/src/ceph-volume/ceph_volume/decorators.py index c1e14bc79..d0be93817 100644 --- a/ceph/src/ceph-volume/ceph_volume/decorators.py +++ b/ceph/src/ceph-volume/ceph_volume/decorators.py @@ -58,6 +58,9 @@ def catches(catch=None, handler=None, exit=True): try: return f(*a, **kw) except catch as e: + import logging + logger = logging.getLogger('ceph_volume') + logger.exception('exception caught by decorator') if os.environ.get('CEPH_VOLUME_DEBUG'): raise if handler: diff --git a/ceph/src/ceph-volume/ceph_volume/devices/__init__.py b/ceph/src/ceph-volume/ceph_volume/devices/__init__.py index c77c344d6..8af5d1e74 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/__init__.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/__init__.py @@ -1 +1 @@ -from . import lvm # noqa +from . import lvm, simple # noqa diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py index 5a755672a..0a50e7a33 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/activate.py @@ -1,16 +1,25 @@ from __future__ import print_function import argparse +import logging +import os from textwrap import dedent from ceph_volume import process, conf, decorators from ceph_volume.util import system, disk +from ceph_volume.util import prepare as prepare_utils from ceph_volume.systemd import systemctl -from . import api +from ceph_volume.api import lvm as api + + +logger = logging.getLogger(__name__) def activate_filestore(lvs): # find the osd osd_lv = lvs.get(lv_tags={'ceph.type': 'data'}) + if not osd_lv: + raise RuntimeError('Unable to find a data LV for filestore activation') osd_id = osd_lv.tags['ceph.osd_id'] + conf.cluster = osd_lv.tags['ceph.cluster_name'] # it may have a volume with a journal osd_journal_lv = lvs.get(lv_tags={'ceph.type': 'journal'}) # TODO: add sensible error reporting if this is ever the case @@ -29,7 +38,7 @@ def activate_filestore(lvs): # mount the osd source = osd_lv.lv_path destination = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id) - if not system.is_mounted(source, destination=destination): + if not system.device_is_mounted(source, destination=destination): process.run(['sudo', 'mount', '-v', source, destination]) # always re-do the symlink regardless if it exists, so that the journal @@ -47,9 +56,76 @@ def activate_filestore(lvs): systemctl.start_osd(osd_id) +def get_osd_device_path(osd_lv, lvs, device_type): + """ + ``device_type`` can be one of ``db``, ``wal`` or ``block`` so that + we can query ``lvs`` (a ``Volumes`` object) and fallback to querying the uuid + if that is not present. + + Return a path if possible, failing to do that a ``None``, since some of these devices + are optional + """ + osd_lv = lvs.get(lv_tags={'ceph.type': 'block'}) + uuid_tag = 'ceph.%s_uuid' % device_type + device_uuid = osd_lv.tags.get(uuid_tag) + if not device_uuid: + return None + + device_lv = lvs.get(lv_uuid=device_uuid) + if device_lv: + return device_lv.lv_path + else: + # this could be a regular device, so query it with blkid + physical_device = disk.get_device_from_partuuid(device_uuid) + return physical_device or None + return None + + def activate_bluestore(lvs): - # TODO - pass + # find the osd + osd_lv = lvs.get(lv_tags={'ceph.type': 'block'}) + osd_id = osd_lv.tags['ceph.osd_id'] + conf.cluster = osd_lv.tags['ceph.cluster_name'] + osd_fsid = osd_lv.tags['ceph.osd_fsid'] + db_device_path = get_osd_device_path(osd_lv, lvs, 'db') + wal_device_path = get_osd_device_path(osd_lv, lvs, 'wal') + + # mount on tmpfs the osd directory + osd_path = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id) + if not system.path_is_mounted(osd_path): + # mkdir -p and mount as tmpfs + prepare_utils.create_osd_path(osd_id, tmpfs=True) + # XXX This needs to be removed once ceph-bluestore-tool can deal with + # symlinks that exist in the osd dir + for link_name in ['block', 'block.db', 'block.wal']: + link_path = os.path.join(osd_path, link_name) + if os.path.exists(link_path): + os.unlink(os.path.join(osd_path, link_name)) + # Once symlinks are removed, the osd dir can be 'primed again. + process.run([ + 'sudo', 'ceph-bluestore-tool', '--cluster=%s' % conf.cluster, + 'prime-osd-dir', '--dev', osd_lv.lv_path, + '--path', osd_path]) + # always re-do the symlink regardless if it exists, so that the block, + # block.wal, and block.db devices that may have changed can be mapped + # correctly every time + process.run(['sudo', 'ln', '-snf', osd_lv.lv_path, os.path.join(osd_path, 'block')]) + system.chown(os.path.join(osd_path, 'block')) + system.chown(osd_path) + if db_device_path: + destination = os.path.join(osd_path, 'block.db') + process.run(['sudo', 'ln', '-snf', db_device_path, destination]) + system.chown(db_device_path) + if wal_device_path: + destination = os.path.join(osd_path, 'block.wal') + process.run(['sudo', 'ln', '-snf', wal_device_path, destination]) + system.chown(wal_device_path) + + # enable the ceph-volume unit for this OSD + systemctl.enable_volume(osd_id, osd_fsid, 'lvm') + + # start the OSD + systemctl.start_osd(osd_id) class Activate(object): @@ -69,7 +145,22 @@ class Activate(object): lvs.filter(lv_tags={'ceph.osd_fsid': args.osd_fsid}) if not lvs: raise RuntimeError('could not find osd.%s with fsid %s' % (args.osd_id, args.osd_fsid)) - activate_filestore(lvs) + # This argument is only available when passed in directly or via + # systemd, not when ``create`` is being used + if getattr(args, 'auto_detect_objectstore', False): + logger.info('auto detecting objectstore') + # may get multiple lvs, so can't do lvs.get() calls here + for lv in lvs: + has_journal = lv.tags.get('ceph.journal_uuid') + if has_journal: + logger.info('found a journal associated with the OSD, assuming filestore') + return activate_filestore(lvs) + logger.info('unable to find a journal associated with the OSD, assuming bluestore') + return activate_bluestore(lvs) + if args.bluestore: + activate_bluestore(lvs) + elif args.filestore: + activate_filestore(lvs) def main(self): sub_command_help = dedent(""" @@ -100,18 +191,27 @@ class Activate(object): nargs='?', help='The FSID of the OSD, similar to a SHA1' ) + parser.add_argument( + '--auto-detect-objectstore', + action='store_true', + help='Autodetect the objectstore by inspecting the OSD', + ) parser.add_argument( '--bluestore', - action='store_true', default=False, + action='store_true', help='filestore objectstore (not yet implemented)', ) parser.add_argument( '--filestore', - action='store_true', default=True, + action='store_true', help='filestore objectstore (current default)', ) if len(self.argv) == 0: print(sub_command_help) return args = parser.parse_args(self.argv) + # Default to bluestore here since defaulting it in add_argument may + # cause both to be True + if not args.bluestore and not args.filestore: + args.bluestore = True self.activate(args) diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/common.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/common.py index b4e4ee3ad..b2fbbf991 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/common.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/common.py @@ -15,30 +15,30 @@ def common_parser(prog, description): required_args = parser.add_argument_group('required arguments') parser.add_argument( '--journal', - help='A logical volume (vg_name/lv_name), or path to a device', + help='(filestore) A logical volume (vg_name/lv_name), or path to a device', ) required_args.add_argument( '--data', required=True, type=arg_validators.LVPath(), - help='A logical volume (vg_name/lv_name) for OSD data', + help='OSD data path. A physical device or logical volume', ) parser.add_argument( '--journal-size', default=5, metavar='GB', type=int, - help='Size (in GB) A logical group name or a path to a logical volume', + help='(filestore) Size (in GB) for the journal', ) parser.add_argument( '--bluestore', - action='store_true', default=False, - help='Use the bluestore objectstore (not currently supported)', + action='store_true', + help='Use the bluestore objectstore', ) parser.add_argument( '--filestore', - action='store_true', default=True, - help='Use the filestore objectstore (currently the only supported object store)', + action='store_true', + help='Use the filestore objectstore', ) parser.add_argument( '--osd-id', @@ -48,6 +48,16 @@ def common_parser(prog, description): '--osd-fsid', help='Reuse an existing OSD fsid', ) + parser.add_argument( + '--block.db', + dest='block_db', + help='(bluestore) Path to bluestore block.db logical volume or device', + ) + parser.add_argument( + '--block.wal', + dest='block_wal', + help='(bluestore) Path to bluestore block.wal logical volume or device', + ) # Do not parse args, so that consumers can do something before the args get # parsed triggering argparse behavior return parser diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/create.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/create.py index 8c747f342..353b26ab4 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/create.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/create.py @@ -50,4 +50,8 @@ class Create(object): print(sub_command_help) return args = parser.parse_args(self.argv) + # Default to bluestore here since defaulting it in add_argument may + # cause both to be True + if args.bluestore is None and args.filestore is None: + args.bluestore = True self.create(args) diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/listing.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/listing.py new file mode 100644 index 000000000..6982f91bc --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/listing.py @@ -0,0 +1,244 @@ +from __future__ import print_function +import argparse +import json +import logging +from textwrap import dedent +from ceph_volume import decorators +from ceph_volume.util import disk +from ceph_volume.api import lvm as api + +logger = logging.getLogger(__name__) + + +osd_list_header_template = """\n +{osd_id:=^20}""" + + +osd_device_header_template = """ + + [{type: >4}] {path} +""" + +device_metadata_item_template = """ + {tag_name: <25} {value}""" + + +def readable_tag(tag): + actual_name = tag.split('.')[-1] + return actual_name.replace('_', ' ') + + +def pretty_report(report): + output = [] + for _id, devices in report.items(): + output.append( + osd_list_header_template.format(osd_id=" osd.%s " % _id) + ) + for device in devices: + output.append( + osd_device_header_template.format( + type=device['type'], + path=device['path'] + ) + ) + for tag_name, value in device.get('tags', {}).items(): + output.append( + device_metadata_item_template.format( + tag_name=readable_tag(tag_name), + value=value + ) + ) + print(''.join(output)) + + +class List(object): + + help = 'list logical volumes and devices associated with Ceph' + + def __init__(self, argv): + self.argv = argv + + @decorators.needs_root + def list(self, args): + # ensure everything is up to date before calling out + # to list lv's + self.update() + report = self.generate(args) + if args.format == 'json': + # If the report is empty, we don't return a non-zero exit status + # because it is assumed this is going to be consumed by automated + # systems like ceph-ansible which would be forced to ignore the + # non-zero exit status if all they need is the information in the + # JSON object + print(json.dumps(report, indent=4, sort_keys=True)) + else: + if not report: + raise SystemExit('No valid Ceph devices found') + pretty_report(report) + + def update(self): + """ + Ensure all journal devices are up to date if they aren't a logical + volume + """ + lvs = api.Volumes() + for lv in lvs: + try: + lv.tags['ceph.osd_id'] + except KeyError: + # only consider ceph-based logical volumes, everything else + # will get ignored + continue + + for device_type in ['journal', 'block', 'wal', 'db']: + device_name = 'ceph.%s_device' % device_type + device_uuid = lv.tags.get('ceph.%s_uuid' % device_type) + if not device_uuid: + # bluestore will not have a journal, filestore will not have + # a block/wal/db, so we must skip if not present + continue + disk_device = disk.get_device_from_partuuid(device_uuid) + if disk_device: + if lv.tags[device_name] != disk_device: + # this means that the device has changed, so it must be updated + # on the API to reflect this + lv.set_tags({device_name: disk_device}) + + def generate(self, args): + """ + Generate reports for an individual device or for all Ceph-related + devices, logical or physical, as long as they have been prepared by + this tool before and contain enough metadata. + """ + if args.device: + return self.single_report(args.device) + else: + return self.full_report() + + def single_report(self, device): + """ + Generate a report for a single device. This can be either a logical + volume in the form of vg/lv or a device with an absolute path like + /dev/sda1 + """ + lvs = api.Volumes() + report = {} + lv = api.get_lv_from_argument(device) + if lv: + try: + _id = lv.tags['ceph.osd_id'] + except KeyError: + logger.warning('device is not part of ceph: %s', device) + return report + + report.setdefault(_id, []) + report[_id].append( + lv.as_dict() + ) + + else: + # this has to be a journal/wal/db device (not a logical volume) so try + # to find the PARTUUID that should be stored in the OSD logical + # volume + for device_type in ['journal', 'block', 'wal', 'db']: + device_tag_name = 'ceph.%s_device' % device_type + device_tag_uuid = 'ceph.%s_uuid' % device_type + associated_lv = lvs.get(lv_tags={device_tag_name: device}) + if associated_lv: + _id = associated_lv.tags['ceph.osd_id'] + uuid = associated_lv.tags[device_tag_uuid] + + report.setdefault(_id, []) + report[_id].append( + { + 'tags': {'PARTUUID': uuid}, + 'type': device_type, + 'path': device, + } + ) + return report + + def full_report(self): + """ + Generate a report for all the logical volumes and associated devices + that have been previously prepared by Ceph + """ + lvs = api.Volumes() + report = {} + for lv in lvs: + try: + _id = lv.tags['ceph.osd_id'] + except KeyError: + # only consider ceph-based logical volumes, everything else + # will get ignored + continue + + report.setdefault(_id, []) + report[_id].append( + lv.as_dict() + ) + + for device_type in ['journal', 'block', 'wal', 'db']: + device_uuid = lv.tags.get('ceph.%s_uuid' % device_type) + if not device_uuid: + # bluestore will not have a journal, filestore will not have + # a block/wal/db, so we must skip if not present + continue + if not api.get_lv(lv_uuid=device_uuid): + # means we have a regular device, so query blkid + disk_device = disk.get_device_from_partuuid(device_uuid) + if disk_device: + report[_id].append( + { + 'tags': {'PARTUUID': device_uuid}, + 'type': device_type, + 'path': disk_device, + } + ) + + return report + + def main(self): + sub_command_help = dedent(""" + List devices or logical volumes associated with Ceph. An association is + determined if a device has information relating to an OSD. This is + verified by querying LVM's metadata and correlating it with devices. + + The lvs associated with the OSD need to have been prepared previously, + so that all needed tags and metadata exist. + + Full listing of all system devices associated with a cluster:: + + ceph-volume lvm list + + List a particular device, reporting all metadata about it:: + + ceph-volume lvm list /dev/sda1 + + List a logical volume, along with all its metadata (vg is a volume + group, and lv the logical volume name):: + + ceph-volume lvm list {vg/lv} + """) + parser = argparse.ArgumentParser( + prog='ceph-volume lvm list', + formatter_class=argparse.RawDescriptionHelpFormatter, + description=sub_command_help, + ) + + parser.add_argument( + 'device', + metavar='DEVICE', + nargs='?', + help='Path to an lv (as vg/lv) or to a device like /dev/sda1' + ) + + parser.add_argument( + '--format', + help='output format, defaults to "pretty"', + default='pretty', + choices=['json', 'pretty'], + ) + + args = parser.parse_args(self.argv) + self.list(args) diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/main.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/main.py index 59e69329b..8b698a03f 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/main.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/main.py @@ -5,6 +5,8 @@ from . import activate from . import prepare from . import create from . import trigger +from . import listing +from . import zap class LVM(object): @@ -22,6 +24,8 @@ class LVM(object): 'prepare': prepare.Prepare, 'create': create.Create, 'trigger': trigger.Trigger, + 'list': listing.List, + 'zap': zap.Zap, } def __init__(self, argv): diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/prepare.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/prepare.py index 1ca5b0d88..5a7daa3dc 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/prepare.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/prepare.py @@ -1,17 +1,17 @@ from __future__ import print_function import json -import os +import uuid from textwrap import dedent from ceph_volume.util import prepare as prepare_utils from ceph_volume.util import system, disk from ceph_volume import conf, decorators, terminal -from . import api +from ceph_volume.api import lvm as api from .common import prepare_parser def prepare_filestore(device, journal, secrets, id_=None, fsid=None): """ - :param device: The name of the volume group or lvm to work with + :param device: The name of the logical volume to work with :param journal: similar to device but can also be a regular/plain disk :param secrets: A dict with the secrets needed to create the osd (e.g. cephx) :param id_: The OSD id @@ -25,7 +25,7 @@ def prepare_filestore(device, journal, secrets, id_=None, fsid=None): # allow re-using an id, in case a prepare failed osd_id = id_ or prepare_utils.create_id(fsid, json_secrets) # create the directory - prepare_utils.create_path(osd_id) + prepare_utils.create_osd_path(osd_id) # format the device prepare_utils.format_device(device) # mount the data device @@ -35,13 +35,42 @@ def prepare_filestore(device, journal, secrets, id_=None, fsid=None): # get the latest monmap prepare_utils.get_monmap(osd_id) # prepare the osd filesystem - prepare_utils.osd_mkfs(osd_id, fsid) + prepare_utils.osd_mkfs_filestore(osd_id, fsid) # write the OSD keyring if it doesn't exist already prepare_utils.write_keyring(osd_id, cephx_secret) -def prepare_bluestore(): - raise NotImplemented() +def prepare_bluestore(block, wal, db, secrets, id_=None, fsid=None): + """ + :param block: The name of the logical volume for the bluestore data + :param wal: a regular/plain disk or logical volume, to be used for block.wal + :param db: a regular/plain disk or logical volume, to be used for block.db + :param secrets: A dict with the secrets needed to create the osd (e.g. cephx) + :param id_: The OSD id + :param fsid: The OSD fsid, also known as the OSD UUID + """ + cephx_secret = secrets.get('cephx_secret', prepare_utils.create_key()) + json_secrets = json.dumps(secrets) + + # allow re-using an existing fsid, in case prepare failed + fsid = fsid or system.generate_uuid() + # allow re-using an id, in case a prepare failed + osd_id = id_ or prepare_utils.create_id(fsid, json_secrets) + # create the directory + prepare_utils.create_osd_path(osd_id, tmpfs=True) + # symlink the block + prepare_utils.link_block(block, osd_id) + # get the latest monmap + prepare_utils.get_monmap(osd_id) + # write the OSD keyring if it doesn't exist already + prepare_utils.write_keyring(osd_id, cephx_secret) + # prepare the osd filesystem + prepare_utils.osd_mkfs_bluestore( + osd_id, fsid, + keyring=cephx_secret, + wal=wal, + db=db + ) class Prepare(object): @@ -51,19 +80,20 @@ class Prepare(object): def __init__(self, argv): self.argv = argv - def get_journal_ptuuid(self, argument): + def get_ptuuid(self, argument): uuid = disk.get_partuuid(argument) if not uuid: terminal.error('blkid could not detect a PARTUUID for device: %s' % argument) - raise RuntimeError('unable to use device for a journal') + raise RuntimeError('unable to use device') return uuid - def get_journal_lv(self, argument): + def get_lv(self, argument): """ - Perform some parsing of the value of ``--journal`` so that the process - can determine correctly if it got a device path or an lv - :param argument: The value of ``--journal``, that will need to be split - to retrieve the actual lv + Perform some parsing of the command-line value so that the process + can determine correctly if it got a device path or an lv. + + :param argument: The command-line value that will need to be split to + retrieve the actual lv """ try: vg_name, lv_name = argument.split('/') @@ -71,6 +101,66 @@ class Prepare(object): return None return api.get_lv(lv_name=lv_name, vg_name=vg_name) + def setup_device(self, device_type, device_name, tags): + """ + Check if ``device`` is an lv, if so, set the tags, making sure to + update the tags with the lv_uuid and lv_path which the incoming tags + will not have. + + If the device is not a logical volume, then retrieve the partition UUID + by querying ``blkid`` + """ + if device_name is None: + return '', '', tags + tags['ceph.type'] = device_type + lv = self.get_lv(device_name) + if lv: + uuid = lv.lv_uuid + path = lv.lv_path + tags['ceph.%s_uuid' % device_type] = uuid + tags['ceph.%s_device' % device_type] = path + lv.set_tags(tags) + else: + # otherwise assume this is a regular disk partition + uuid = self.get_ptuuid(device_name) + path = device_name + tags['ceph.%s_uuid' % device_type] = uuid + tags['ceph.%s_device' % device_type] = path + return path, uuid, tags + + def prepare_device(self, arg, device_type, cluster_fsid, osd_fsid): + """ + Check if ``arg`` is a device or partition to create an LV out of it + with a distinct volume group name, assigning LV tags on it and + ultimately, returning the logical volume object. Failing to detect + a device or partition will result in error. + + :param arg: The value of ``--data`` when parsing args + :param device_type: Usually, either ``data`` or ``block`` (filestore vs. bluestore) + :param cluster_fsid: The cluster fsid/uuid + :param osd_fsid: The OSD fsid/uuid + """ + if disk.is_partition(arg) or disk.is_device(arg): + # we must create a vg, and then a single lv + vg_name = "ceph-%s" % cluster_fsid + if api.get_vg(vg_name=vg_name): + # means we already have a group for this, make a different one + # XXX this could end up being annoying for an operator, maybe? + vg_name = "ceph-%s" % str(uuid.uuid4()) + api.create_vg(vg_name, arg) + lv_name = "osd-%s-%s" % (device_type, osd_fsid) + return api.create_lv( + lv_name, + vg_name, # the volume group + tags={'ceph.type': device_type}) + else: + error = [ + 'Cannot use device (%s).', + 'A vg/lv path or an existing device is needed' % arg] + raise RuntimeError(' '.join(error)) + + raise RuntimeError('no data logical volume found with: %s' % arg) + @decorators.needs_root def prepare(self, args): # FIXME we don't allow re-using a keyring, we always generate one for the @@ -80,69 +170,66 @@ class Prepare(object): secrets = {'cephx_secret': prepare_utils.create_key()} cluster_fsid = conf.ceph.get('global', 'fsid') - fsid = args.osd_fsid or system.generate_uuid() - #osd_id = args.osd_id or prepare_utils.create_id(fsid) + osd_fsid = args.osd_fsid or system.generate_uuid() # allow re-using an id, in case a prepare failed - osd_id = args.osd_id or prepare_utils.create_id(fsid, json.dumps(secrets)) - vg_name, lv_name = args.data.split('/') + osd_id = args.osd_id or prepare_utils.create_id(osd_fsid, json.dumps(secrets)) if args.filestore: - data_lv = api.get_lv(lv_name=lv_name, vg_name=vg_name) - - # we must have either an existing data_lv or a newly created, so lets make - # sure that the tags are correct - if not data_lv: - raise RuntimeError('no data logical volume found with: %s' % args.data) - if not args.journal: raise RuntimeError('--journal is required when using --filestore') - journal_lv = self.get_journal_lv(args.journal) - if journal_lv: - journal_device = journal_lv.lv_path - journal_uuid = journal_lv.lv_uuid - # we can only set tags on an lv, the pv (if any) can't as we - # aren't making it part of an lvm group (vg) - journal_lv.set_tags({ - 'ceph.type': 'journal', - 'ceph.osd_fsid': fsid, - 'ceph.osd_id': osd_id, - 'ceph.cluster_fsid': cluster_fsid, - 'ceph.journal_device': journal_device, - 'ceph.journal_uuid': journal_uuid, - 'ceph.data_device': data_lv.lv_path, - 'ceph.data_uuid': data_lv.lv_uuid, - }) - - # allow a file - elif os.path.isfile(args.journal): - journal_uuid = '' - journal_device = args.journal - - # otherwise assume this is a regular disk partition - else: - journal_uuid = self.get_journal_ptuuid(args.journal) - journal_device = args.journal + data_lv = self.get_lv(args.data) + if not data_lv: + data_lv = self.prepare_device(args.data, 'data', cluster_fsid, osd_fsid) - data_lv.set_tags({ - 'ceph.type': 'data', - 'ceph.osd_fsid': fsid, + tags = { + 'ceph.osd_fsid': osd_fsid, 'ceph.osd_id': osd_id, 'ceph.cluster_fsid': cluster_fsid, - 'ceph.journal_device': journal_device, - 'ceph.journal_uuid': journal_uuid, + 'ceph.cluster_name': conf.cluster, 'ceph.data_device': data_lv.lv_path, 'ceph.data_uuid': data_lv.lv_uuid, - }) + } + + journal_device, journal_uuid, tags = self.setup_device('journal', args.journal, tags) + + tags['ceph.type'] = 'data' + data_lv.set_tags(tags) prepare_filestore( data_lv.lv_path, journal_device, secrets, id_=osd_id, - fsid=fsid, + fsid=osd_fsid, ) elif args.bluestore: - prepare_bluestore(args) + block_lv = self.get_lv(args.data) + if not block_lv: + block_lv = self.prepare_device(args.data, 'block', cluster_fsid, osd_fsid) + + tags = { + 'ceph.osd_fsid': osd_fsid, + 'ceph.osd_id': osd_id, + 'ceph.cluster_fsid': cluster_fsid, + 'ceph.cluster_name': conf.cluster, + 'ceph.block_device': block_lv.lv_path, + 'ceph.block_uuid': block_lv.lv_uuid, + } + + wal_device, wal_uuid, tags = self.setup_device('wal', args.block_wal, tags) + db_device, db_uuid, tags = self.setup_device('db', args.block_db, tags) + + tags['ceph.type'] = 'block' + block_lv.set_tags(tags) + + prepare_bluestore( + block_lv.lv_path, + wal_device, + db_device, + secrets, + id_=osd_id, + fsid=osd_fsid, + ) def main(self): sub_command_help = dedent(""" @@ -166,17 +253,30 @@ class Prepare(object): Existing logical volume (lv) or device: - ceph-volume lvm prepare --data {logical volume} --journal /path/to/{lv}|{device} + ceph-volume lvm prepare --filestore --data {vg/lv} --journal /path/to/device Or: - ceph-volume lvm prepare --data {data volume group} --journal {journal volume group} + ceph-volume lvm prepare --filestore --data {vg/lv} --journal {vg/lv} + + Existing block device, that will be made a group and logical volume: + + ceph-volume lvm prepare --filestore --data /path/to/device --journal {vg/lv} + + Bluestore + --------- + + Existing logical volume (lv): + + ceph-volume lvm prepare --bluestore --data {vg/lv} + + Existing block device, that will be made a group and logical volume: - Collocated (same group) for data and journal - -------------------------------------------- + ceph-volume lvm prepare --bluestore --data /path/to/device - ceph-volume lvm prepare --data {volume group} + Optionally, can consume db and wal devices or logical volumes: + ceph-volume lvm prepare --bluestore --data {vg/lv} --block.wal {device} --block-db {vg/lv} """) parser = prepare_parser( prog='ceph-volume lvm prepare', @@ -186,4 +286,8 @@ class Prepare(object): print(sub_command_help) return args = parser.parse_args(self.argv) + # Default to bluestore here since defaulting it in add_argument may + # cause both to be True + if args.bluestore is None and args.filestore is None: + args.bluestore = True self.prepare(args) diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/trigger.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/trigger.py index 911162072..dc57011df 100644 --- a/ceph/src/ceph-volume/ceph_volume/devices/lvm/trigger.py +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/trigger.py @@ -67,4 +67,4 @@ class Trigger(object): args = parser.parse_args(self.argv) osd_id = parse_osd_id(args.systemd_data) osd_uuid = parse_osd_uuid(args.systemd_data) - Activate([osd_id, osd_uuid]).main() + Activate(['--auto-detect-objectstore', osd_id, osd_uuid]).main() diff --git a/ceph/src/ceph-volume/ceph_volume/devices/lvm/zap.py b/ceph/src/ceph-volume/ceph_volume/devices/lvm/zap.py new file mode 100644 index 000000000..df19686ff --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/devices/lvm/zap.py @@ -0,0 +1,107 @@ +import argparse +import logging + +from textwrap import dedent + +from ceph_volume import decorators, terminal, process +from ceph_volume.api import lvm as api + +logger = logging.getLogger(__name__) + + +def wipefs(path): + """ + Removes the filesystem from an lv or partition. + """ + process.run([ + 'sudo', + 'wipefs', + '--all', + path + ]) + + +def zap_data(path): + """ + Clears all data from the given path. Path should be + an absolute path to an lv or partition. + + 10M of data is written to the path to make sure that + there is no trace left of any previous Filesystem. + """ + process.run([ + 'dd', + 'if=/dev/zero', + 'of={path}'.format(path=path), + 'bs=1M', + 'count=10', + ]) + + +class Zap(object): + + help = 'Removes all data and filesystems from a logical volume or partition.' + + def __init__(self, argv): + self.argv = argv + + @decorators.needs_root + def zap(self, args): + device = args.device + lv = api.get_lv_from_argument(device) + if lv: + # we are zapping a logical volume + path = lv.lv_path + else: + # we are zapping a partition + #TODO: ensure device is a partition + path = device + + logger.info("Zapping: %s", path) + terminal.write("Zapping: %s" % path) + + wipefs(path) + zap_data(path) + + if lv: + # remove all lvm metadata + lv.clear_tags() + + terminal.success("Zapping successful for: %s" % path) + + def main(self): + sub_command_help = dedent(""" + Zaps the given logical volume or partition. If given a path to a logical + volume it must be in the format of vg/lv. Any filesystems present + on the given lv or partition will be removed and all data will be purged. + + However, the lv or partition will be kept intact. + + Example calls for supported scenarios: + + Zapping a logical volume: + + ceph-volume lvm zap {vg name/lv name} + + Zapping a partition: + + ceph-volume lvm zap /dev/sdc1 + + """) + parser = argparse.ArgumentParser( + prog='ceph-volume lvm zap', + formatter_class=argparse.RawDescriptionHelpFormatter, + description=sub_command_help, + ) + + parser.add_argument( + 'device', + metavar='DEVICE', + nargs='?', + help='Path to an lv (as vg/lv) or to a partition like /dev/sda1' + ) + if len(self.argv) == 0: + print(sub_command_help) + return + args = parser.parse_args(self.argv) + self.zap(args) diff --git a/ceph/src/ceph-volume/ceph_volume/devices/simple/__init__.py b/ceph/src/ceph-volume/ceph_volume/devices/simple/__init__.py new file mode 100644 index 000000000..280e130ed --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/devices/simple/__init__.py @@ -0,0 +1 @@ +from .main import Simple # noqa diff --git a/ceph/src/ceph-volume/ceph_volume/devices/simple/activate.py b/ceph/src/ceph-volume/ceph_volume/devices/simple/activate.py new file mode 100644 index 000000000..fdc50f0fa --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/devices/simple/activate.py @@ -0,0 +1,152 @@ +from __future__ import print_function +import argparse +import json +import logging +import os +from textwrap import dedent +from ceph_volume import process, decorators, terminal +from ceph_volume.util import system, disk +from ceph_volume.systemd import systemctl + + +logger = logging.getLogger(__name__) + + +class Activate(object): + + help = 'Enable systemd units to mount configured devices and start a Ceph OSD' + + def __init__(self, argv, systemd=False): + self.argv = argv + self.systemd = systemd + + @decorators.needs_root + def activate(self, args): + with open(args.json_config, 'r') as fp: + osd_metadata = json.load(fp) + + osd_id = osd_metadata.get('whoami', args.osd_id) + osd_fsid = osd_metadata.get('fsid', args.osd_fsid) + + cluster_name = osd_metadata.get('cluster_name', 'ceph') + osd_dir = '/var/lib/ceph/osd/%s-%s' % (cluster_name, osd_id) + data_uuid = osd_metadata.get('data', {}).get('uuid') + if not data_uuid: + raise RuntimeError( + 'Unable to activate OSD %s - no "uuid" key found for data' % args.osd_id + ) + data_device = disk.get_device_from_partuuid(data_uuid) + journal_device = disk.get_device_from_partuuid(osd_metadata.get('journal', {}).get('uuid')) + block_device = disk.get_device_from_partuuid(osd_metadata.get('block', {}).get('uuid')) + block_db_device = disk.get_device_from_partuuid(osd_metadata.get('block.db', {}).get('uuid')) + block_wal_device = disk.get_device_from_partuuid( + osd_metadata.get('block.wal', {}).get('uuid') + ) + + if not system.device_is_mounted(data_device, destination=osd_dir): + process.run(['sudo', 'mount', '-v', data_device, osd_dir]) + + device_map = { + 'journal': journal_device, + 'block': block_device, + 'block.db': block_db_device, + 'block.wal': block_wal_device + } + + for name, device in device_map.items(): + if not device: + continue + # always re-do the symlink regardless if it exists, so that the journal + # device path that may have changed can be mapped correctly every time + destination = os.path.join(osd_dir, name) + process.run(['sudo', 'ln', '-snf', device, destination]) + + # make sure that the journal has proper permissions + system.chown(device) + + if not self.systemd: + # enable the ceph-volume unit for this OSD + systemctl.enable_volume(osd_id, osd_fsid, 'simple') + + # disable any/all ceph-disk units + systemctl.mask_ceph_disk() + + # enable the OSD + systemctl.enable_osd(osd_id) + + # start the OSD + systemctl.start_osd(osd_id) + + if not self.systemd: + terminal.success('Successfully activated OSD %s with FSID %s' % (osd_id, osd_fsid)) + terminal.warning( + ('All ceph-disk systemd units have been disabled to ' + 'prevent OSDs getting triggered by UDEV events') + ) + + def main(self): + sub_command_help = dedent(""" + Activate OSDs by mounting devices previously configured to their + appropriate destination:: + + ceph-volume simple activate {ID} {FSID} + + Or using a JSON file directly:: + + ceph-volume simple activate --file /etc/ceph/osd/{ID}-{FSID}.json + + The OSD must have been "scanned" previously (see ``ceph-volume simple + scan``), so that all needed OSD device information and metadata exist. + + A previously scanned OSD would exist like:: + + /etc/ceph/osd/{ID}-{FSID}.json + + + Environment variables supported: + + CEPH_VOLUME_SIMPLE_JSON_DIR: Directory location for scanned OSD JSON configs + """) + parser = argparse.ArgumentParser( + prog='ceph-volume simple activate', + formatter_class=argparse.RawDescriptionHelpFormatter, + description=sub_command_help, + ) + parser.add_argument( + 'osd_id', + metavar='ID', + nargs='?', + help='The ID of the OSD, usually an integer, like 0' + ) + parser.add_argument( + 'osd_fsid', + metavar='FSID', + nargs='?', + help='The FSID of the OSD, similar to a SHA1' + ) + parser.add_argument( + '--file', + help='The path to a JSON file, from a scanned OSD' + ) + if len(self.argv) == 0: + print(sub_command_help) + return + args = parser.parse_args(self.argv) + if not args.file: + if not args.osd_id and not args.osd_fsid: + terminal.error('ID and FSID are required to find the right OSD to activate') + terminal.error('from a scanned OSD location in /etc/ceph/osd/') + raise RuntimeError('Unable to activate without both ID and FSID') + # don't allow a CLI flag to specify the JSON dir, because that might + # implicitly indicate that it would be possible to activate a json file + # at a non-default location which would not work at boot time if the + # custom location is not exposed through an ENV var + json_dir = os.environ.get('CEPH_VOLUME_SIMPLE_JSON_DIR', '/etc/ceph/osd/') + if args.file: + json_config = args.file + else: + json_config = os.path.join(json_dir, '%s-%s.json' % (args.osd_id, args.osd_fsid)) + if not os.path.exists(json_config): + raise RuntimeError('Expected JSON config path not found: %s' % json_config) + args.json_config = json_config + self.activate(args) diff --git a/ceph/src/ceph-volume/ceph_volume/devices/simple/main.py b/ceph/src/ceph-volume/ceph_volume/devices/simple/main.py new file mode 100644 index 000000000..2119963f8 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/devices/simple/main.py @@ -0,0 +1,41 @@ +import argparse +from textwrap import dedent +from ceph_volume import terminal +from . import scan +from . import activate +from . import trigger + + +class Simple(object): + + help = 'Manage already deployed OSDs with ceph-volume' + + _help = dedent(""" + Take over a deployed OSD, persisting its metadata in /etc/ceph/osd/ so that it can be managed + with ceph-volume directly. Avoids UDEV and ceph-disk handling. + + {sub_help} + """) + + mapper = { + 'scan': scan.Scan, + 'activate': activate.Activate, + 'trigger': trigger.Trigger, + } + + def __init__(self, argv): + self.argv = argv + + def print_help(self, sub_help): + return self._help.format(sub_help=sub_help) + + def main(self): + terminal.dispatch(self.mapper, self.argv) + parser = argparse.ArgumentParser( + prog='ceph-volume simple', + formatter_class=argparse.RawDescriptionHelpFormatter, + description=self.print_help(terminal.subhelp(self.mapper)), + ) + parser.parse_args(self.argv) + if len(self.argv) <= 1: + return parser.print_help() diff --git a/ceph/src/ceph-volume/ceph_volume/devices/simple/scan.py b/ceph/src/ceph-volume/ceph_volume/devices/simple/scan.py new file mode 100644 index 000000000..905baf415 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/devices/simple/scan.py @@ -0,0 +1,206 @@ +from __future__ import print_function +import argparse +import json +import logging +import os +from textwrap import dedent +from ceph_volume import decorators, terminal, conf +from ceph_volume.api import lvm +from ceph_volume.util import arg_validators, system, disk + + +logger = logging.getLogger(__name__) + + +class Scan(object): + + help = 'Capture metadata from an OSD data partition or directory' + + def __init__(self, argv): + self.argv = argv + self._etc_path = '/etc/ceph/osd/' + + @property + def etc_path(self): + if os.path.isdir(self._etc_path): + return self._etc_path + + if not os.path.exists(self._etc_path): + os.mkdir(self._etc_path) + return self._etc_path + + error = "OSD Configuration path (%s) needs to be a directory" % self._etc_path + raise RuntimeError(error) + + def get_contents(self, path): + with open(path, 'r') as fp: + contents = fp.readlines() + if len(contents) > 1: + return ''.join(contents) + return ''.join(contents).strip().strip('\n') + + def scan_device(self, path): + device_metadata = {'path': None, 'uuid': None} + if not path: + return device_metadata + # cannot read the symlink if this is tmpfs + if os.path.islink(path): + device = os.readlink(path) + else: + device = path + lvm_device = lvm.get_lv_from_argument(device) + if lvm_device: + device_uuid = lvm_device.lv_uuid + else: + device_uuid = disk.get_partuuid(device) + + device_metadata['uuid'] = device_uuid + device_metadata['path'] = device + + return device_metadata + + def scan_directory(self, path): + osd_metadata = {'cluster_name': conf.cluster} + path_mounts = system.get_mounts(paths=True) + for _file in os.listdir(path): + file_path = os.path.join(path, _file) + if os.path.islink(file_path): + osd_metadata[_file] = self.scan_device(file_path) + if os.path.isdir(file_path): + continue + # the check for binary needs to go before the file, to avoid + # capturing data from binary files but still be able to capture + # contents from actual files later + if system.is_binary(file_path): + continue + if os.path.isfile(file_path): + osd_metadata[_file] = self.get_contents(file_path) + + device = path_mounts.get(path) + # it is possible to have more than one device, pick the first one, and + # warn that it is possible that more than one device is 'data' + if not device: + terminal.error('Unable to detect device mounted for path: %s' % path) + raise RuntimeError('Cannot activate OSD') + osd_metadata['data'] = self.scan_device(device[0] if len(device) else None) + + return osd_metadata + + @decorators.needs_root + def scan(self, args): + osd_metadata = {'cluster_name': conf.cluster} + device_mounts = system.get_mounts(devices=True) + osd_path = None + logger.info('detecting if argument is a device or a directory: %s', args.osd_path) + if os.path.isdir(args.osd_path): + logger.info('will scan directly, path is a directory') + osd_path = args.osd_path + else: + # assume this is a device, check if it is mounted and use that path + logger.info('path is not a directory, will check if mounted') + if system.device_is_mounted(args.osd_path): + logger.info('argument is a device, which is mounted') + mounted_osd_paths = device_mounts.get(args.osd_path) + osd_path = mounted_osd_paths[0] if len(mounted_osd_paths) else None + + # argument is not a directory, and it is not a device that is mounted + # somewhere so temporarily mount it to poke inside, otherwise, scan + # directly + if not osd_path: + logger.info('device is not mounted, will mount it temporarily to scan') + with system.tmp_mount(args.osd_path) as osd_path: + osd_metadata = self.scan_directory(osd_path) + else: + logger.info('will scan OSD directory at path: %s', osd_path) + osd_metadata = self.scan_directory(osd_path) + + osd_id = osd_metadata['whoami'] + osd_fsid = osd_metadata['fsid'] + filename = '%s-%s.json' % (osd_id, osd_fsid) + json_path = os.path.join(self.etc_path, filename) + if os.path.exists(json_path) and not args.stdout: + if not args.force: + raise RuntimeError( + '--force was not used and OSD metadata file exists: %s' % json_path + ) + + if args.stdout: + print(json.dumps(osd_metadata, indent=4, sort_keys=True, ensure_ascii=False)) + else: + with open(json_path, 'w') as fp: + json.dump(osd_metadata, fp, indent=4, sort_keys=True, ensure_ascii=False) + terminal.success( + 'OSD %s got scanned and metadata persisted to file: %s' % ( + osd_id, + json_path + ) + ) + terminal.success( + 'To take over managment of this scanned OSD, and disable ceph-disk and udev, run:' + ) + terminal.success(' ceph-volume simple activate %s %s' % (osd_id, osd_fsid)) + + if not osd_metadata.get('data'): + msg = 'Unable to determine device mounted on %s' % args.osd_path + logger.warning(msg) + terminal.warning(msg) + terminal.warning('OSD will not be able to start without this information:') + terminal.warning(' "data": "/path/to/device",') + logger.warning('Unable to determine device mounted on %s' % args.osd_path) + + def main(self): + sub_command_help = dedent(""" + Scan an OSD directory for files and configurations that will allow to + take over the management of the OSD. + + Scanned OSDs will get their configurations stored in + /etc/ceph/osd/-.json + + For an OSD ID of 0 with fsid of ``a9d50838-e823-43d6-b01f-2f8d0a77afc2`` + that could mean a scan command that looks like:: + + ceph-volume lvm scan /var/lib/ceph/osd/ceph-0 + + Which would store the metadata in a JSON file at:: + + /etc/ceph/osd/0-a9d50838-e823-43d6-b01f-2f8d0a77afc2.json + + To a scan an existing, running, OSD: + + ceph-volume simple scan /var/lib/ceph/osd/{cluster}-{osd id} + + And to scan a device (mounted or unmounted) that has OSD data in it, for example /dev/sda1 + + ceph-volume simple scan /dev/sda1 + """) + parser = argparse.ArgumentParser( + prog='ceph-volume simple scan', + formatter_class=argparse.RawDescriptionHelpFormatter, + description=sub_command_help, + ) + + parser.add_argument( + '-f', '--force', + action='store_true', + help='If OSD has already been scanned, the JSON file will be overwritten' + ) + + parser.add_argument( + '--stdout', + action='store_true', + help='Do not save to a file, output metadata to stdout' + ) + + parser.add_argument( + 'osd_path', + metavar='OSD_PATH', + type=arg_validators.OSDPath(), + nargs='?', + help='Path to an existing OSD directory or OSD data partition' + ) + + if len(self.argv) == 0: + print(sub_command_help) + return + args = parser.parse_args(self.argv) + self.scan(args) diff --git a/ceph/src/ceph-volume/ceph_volume/devices/simple/trigger.py b/ceph/src/ceph-volume/ceph_volume/devices/simple/trigger.py new file mode 100644 index 000000000..aeb5cf1aa --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/devices/simple/trigger.py @@ -0,0 +1,70 @@ +from __future__ import print_function +import argparse +from textwrap import dedent +from ceph_volume.exceptions import SuffixParsingError +from ceph_volume import decorators +from .activate import Activate + + +def parse_osd_id(string): + osd_id = string.split('-', 1)[0] + if not osd_id: + raise SuffixParsingError('OSD id', string) + if osd_id.isdigit(): + return osd_id + raise SuffixParsingError('OSD id', string) + + +def parse_osd_uuid(string): + osd_id = '%s-' % parse_osd_id(string) + # remove the id first + osd_uuid = string.split(osd_id, 1)[-1] + if not osd_uuid: + raise SuffixParsingError('OSD uuid', string) + return osd_uuid + + +class Trigger(object): + + help = 'systemd helper to activate an OSD' + + def __init__(self, argv): + self.argv = argv + + @decorators.needs_root + def main(self): + sub_command_help = dedent(""" + ** DO NOT USE DIRECTLY ** + This tool is meant to help the systemd unit that knows about OSDs. + + Proxy OSD activation to ``ceph-volume simple activate`` by parsing the + input from systemd, detecting the UUID and ID associated with an OSD:: + + ceph-volume simple trigger {SYSTEMD-DATA} + + The systemd "data" is expected to be in the format of:: + + {OSD ID}-{OSD UUID} + + The devices associated with the OSD need to have been scanned previously, + so that all needed metadata can be used for starting the OSD process. + """) + parser = argparse.ArgumentParser( + prog='ceph-volume simple trigger', + formatter_class=argparse.RawDescriptionHelpFormatter, + description=sub_command_help, + ) + + parser.add_argument( + 'systemd_data', + metavar='SYSTEMD_DATA', + nargs='?', + help='Data from a systemd unit containing ID and UUID of the OSD, like 0-asdf-lkjh' + ) + if len(self.argv) == 0: + print(sub_command_help) + return + args = parser.parse_args(self.argv) + osd_id = parse_osd_id(args.systemd_data) + osd_uuid = parse_osd_uuid(args.systemd_data) + Activate([osd_id, osd_uuid], systemd=True).main() diff --git a/ceph/src/ceph-volume/ceph_volume/main.py b/ceph/src/ceph-volume/ceph_volume/main.py index d4bee154d..e7ed5d88c 100644 --- a/ceph/src/ceph-volume/ceph_volume/main.py +++ b/ceph/src/ceph-volume/ceph_volume/main.py @@ -27,7 +27,7 @@ Ceph Conf: {ceph_path} """ def __init__(self, argv=None, parse=True): - self.mapper = {'lvm': devices.lvm.LVM} + self.mapper = {'lvm': devices.lvm.LVM, 'simple': devices.simple.Simple} self.plugin_help = "No plugins found/loaded" if argv is None: self.argv = sys.argv diff --git a/ceph/src/ceph-volume/ceph_volume/process.py b/ceph/src/ceph-volume/ceph_volume/process.py index bc5047a17..4b6a9c284 100644 --- a/ceph/src/ceph-volume/ceph_volume/process.py +++ b/ceph/src/ceph-volume/ceph_volume/process.py @@ -48,6 +48,47 @@ def log_descriptors(reads, process, terminal_logging): pass +def obfuscate(command_, on=None): + """ + Certain commands that are useful to log might contain information that + should be replaced by '*' like when creating OSDs and the keyryings are + being passed, which should not be logged. + + :param on: A string (will match a flag) or an integer (will match an index) + + If matching on a flag (when ``on`` is a string) it will obfuscate on the + value for that flag. That is a command like ['ls', '-l', '/'] that calls + `obfuscate(command, on='-l')` will obfustace '/' which is the value for + `-l`. + + The reason for `on` to allow either a string or an integer, altering + behavior for both is because it is easier for ``run`` and ``call`` to just + pop a value to obfuscate (vs. allowing an index or a flag) + """ + command = command_[:] + msg = "Running command: %s" % ' '.join(command) + if on in [None, False]: + return msg + + if isinstance(on, int): + index = on + + else: + try: + index = command.index(on) + 1 + except ValueError: + # if the flag just doesn't exist then it doesn't matter just return + # the base msg + return msg + + try: + command[index] = '*' * len(command[index]) + except IndexError: # the index was completely out of range + return msg + + return "Running command: %s" % ' '.join(command) + + def run(command, **kw): """ A real-time-logging implementation of a remote subprocess.Popen call where @@ -57,7 +98,7 @@ def run(command, **kw): :param stop_on_error: If a nonzero exit status is return, it raises a ``RuntimeError`` """ stop_on_error = kw.pop('stop_on_error', True) - command_msg = "Running command: %s" % ' '.join(command) + command_msg = obfuscate(command, kw.pop('obfuscate', None)) stdin = kw.pop('stdin', None) logger.info(command_msg) terminal.write(command_msg) @@ -115,10 +156,12 @@ def call(command, **kw): it is forcefully set to True if a return code is non-zero """ terminal_verbose = kw.pop('terminal_verbose', False) + show_command = kw.pop('show_command', False) command_msg = "Running command: %s" % ' '.join(command) stdin = kw.pop('stdin', None) logger.info(command_msg) - terminal.write(command_msg) + if show_command: + terminal.write(command_msg) process = subprocess.Popen( command, diff --git a/ceph/src/ceph-volume/ceph_volume/systemd/systemctl.py b/ceph/src/ceph-volume/ceph_volume/systemd/systemctl.py index 9bb4d7d3a..ab8f3e70a 100644 --- a/ceph/src/ceph-volume/ceph_volume/systemd/systemctl.py +++ b/ceph/src/ceph-volume/ceph_volume/systemd/systemctl.py @@ -20,6 +20,10 @@ def disable(unit): process.run(['sudo', 'systemctl', 'disable', unit]) +def mask(unit): + process.run(['sudo', 'systemctl', 'mask', unit]) + + def start_osd(id_): return start(osd_unit % id_) @@ -40,9 +44,20 @@ def enable_volume(id_, fsid, device_type='lvm'): return enable(volume_unit % (device_type, id_, fsid)) +def mask_ceph_disk(): + # systemctl allows using a glob like '*' for masking, but there was a bug + # in that it wouldn't allow this for service templates. This means that + # masking ceph-disk@* will not work, so we must link the service directly. + # /etc/systemd takes precendence regardless of the location of the unit + process.run( + ['sudo', 'ln', '-sf', '/dev/null', '/etc/systemd/system/ceph-disk@.service'] + ) + + # # templates # osd_unit = "ceph-osd@%s" +ceph_disk_unit = "ceph-disk@%s" volume_unit = "ceph-volume@%s-%s-%s" diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_api.py b/ceph/src/ceph-volume/ceph_volume/tests/api/test_lvm.py similarity index 84% rename from ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_api.py rename to ceph/src/ceph-volume/ceph_volume/tests/api/test_lvm.py index d6aa54904..3639f01e5 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_api.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/api/test_lvm.py @@ -1,6 +1,6 @@ import pytest from ceph_volume import process, exceptions -from ceph_volume.devices.lvm import api +from ceph_volume.api import lvm as api class TestParseTags(object): @@ -73,6 +73,9 @@ def volumes(monkeypatch): monkeypatch.setattr(process, 'call', lambda x: ('', '', 0)) volumes = api.Volumes() volumes._purge() + # also patch api.Volumes so that when it is called, it will use the newly + # created fixture, with whatever the test method wants to append to it + monkeypatch.setattr(api, 'Volumes', lambda: volumes) return volumes @@ -189,6 +192,20 @@ class TestVolumes(object): with pytest.raises(exceptions.MultipleLVsError): volumes.get(lv_name='foo') + def test_as_dict_infers_type_from_tags(self, volumes): + lv_tags = "ceph.type=data,ceph.fsid=000-aaa" + osd = api.Volume(lv_name='volume1', lv_path='/dev/vg/lv', lv_tags=lv_tags) + volumes.append(osd) + result = volumes.get(lv_tags={'ceph.type': 'data'}).as_dict() + assert result['type'] == 'data' + + def test_as_dict_populates_path_from_lv_api(self, volumes): + lv_tags = "ceph.type=data,ceph.fsid=000-aaa" + osd = api.Volume(lv_name='volume1', lv_path='/dev/vg/lv', lv_tags=lv_tags) + volumes.append(osd) + result = volumes.get(lv_tags={'ceph.type': 'data'}).as_dict() + assert result['path'] == '/dev/vg/lv' + def test_find_the_correct_one(self, volumes): volume1 = api.Volume(lv_name='volume1', lv_path='/dev/vg/lv', lv_tags='') volume2 = api.Volume(lv_name='volume2', lv_path='/dev/vg/lv', lv_tags='') @@ -311,6 +328,47 @@ class TestVolumeGroups(object): volume_groups.filter() +class TestGetLVFromArgument(object): + + def setup(self): + self.foo_volume = api.Volume( + lv_name='foo', lv_path='/path/to/lv', + vg_name='foo_group', lv_tags='' + ) + + def test_non_absolute_path_is_not_valid(self, volumes): + volumes.append(self.foo_volume) + assert api.get_lv_from_argument('foo') is None + + def test_too_many_slashes_is_invalid(self, volumes): + volumes.append(self.foo_volume) + assert api.get_lv_from_argument('path/to/lv') is None + + def test_absolute_path_is_not_lv(self, volumes): + volumes.append(self.foo_volume) + assert api.get_lv_from_argument('/path') is None + + def test_absolute_path_is_lv(self, volumes): + volumes.append(self.foo_volume) + assert api.get_lv_from_argument('/path/to/lv') == self.foo_volume + + +class TestRemoveLV(object): + + def test_removes_lv(self, monkeypatch): + def mock_call(cmd, **kw): + return ('', '', 0) + monkeypatch.setattr(process, 'call', mock_call) + assert api.remove_lv("vg/lv") + + def test_fails_to_remove_lv(self, monkeypatch): + def mock_call(cmd, **kw): + return ('', '', 1) + monkeypatch.setattr(process, 'call', mock_call) + with pytest.raises(RuntimeError): + api.remove_lv("vg/lv") + + class TestCreateLV(object): def setup(self): @@ -320,7 +378,7 @@ class TestCreateLV(object): monkeypatch.setattr(process, 'run', capture) monkeypatch.setattr(process, 'call', capture) monkeypatch.setattr(api, 'get_lv', lambda *a, **kw: self.foo_volume) - api.create_lv('foo', 'foo_group', size=5, type='data') + api.create_lv('foo', 'foo_group', size='5G', tags={'ceph.type': 'data'}) expected = ['sudo', 'lvcreate', '--yes', '-L', '5G', '-n', 'foo', 'foo_group'] assert capture.calls[0]['args'][0] == expected @@ -328,7 +386,7 @@ class TestCreateLV(object): monkeypatch.setattr(process, 'run', capture) monkeypatch.setattr(process, 'call', capture) monkeypatch.setattr(api, 'get_lv', lambda *a, **kw: self.foo_volume) - api.create_lv('foo', 'foo_group', size=5, type='data') + api.create_lv('foo', 'foo_group', size='5G', tags={'ceph.type': 'data'}) ceph_tag = ['sudo', 'lvchange', '--addtag', 'ceph.type=data', '/path'] assert capture.calls[1]['args'][0] == ceph_tag @@ -336,6 +394,6 @@ class TestCreateLV(object): monkeypatch.setattr(process, 'run', capture) monkeypatch.setattr(process, 'call', capture) monkeypatch.setattr(api, 'get_lv', lambda *a, **kw: self.foo_volume) - api.create_lv('foo', 'foo_group', size=5, type='data') + api.create_lv('foo', 'foo_group', size='5G', tags={'ceph.type': 'data'}) data_tag = ['sudo', 'lvchange', '--addtag', 'ceph.data_device=/path', '/path'] assert capture.calls[2]['args'][0] == data_tag diff --git a/ceph/src/ceph-volume/ceph_volume/tests/conftest.py b/ceph/src/ceph-volume/ceph_volume/tests/conftest.py index 7a580e57c..f58033461 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/conftest.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/conftest.py @@ -1,5 +1,7 @@ +import os import pytest -from ceph_volume.devices.lvm import api +from ceph_volume.api import lvm as lvm_api + class Capture(object): @@ -12,6 +14,18 @@ class Capture(object): self.calls.append({'args': a, 'kwargs': kw}) +class Factory(object): + + def __init__(self, **kw): + for k, v in kw.items(): + setattr(self, k, v) + + +@pytest.fixture +def factory(): + return Factory + + @pytest.fixture def capture(): return Capture() @@ -20,7 +34,7 @@ def capture(): @pytest.fixture def volumes(monkeypatch): monkeypatch.setattr('ceph_volume.process.call', lambda x: ('', '', 0)) - volumes = api.Volumes() + volumes = lvm_api.Volumes() volumes._purge() return volumes @@ -28,7 +42,7 @@ def volumes(monkeypatch): @pytest.fixture def volume_groups(monkeypatch): monkeypatch.setattr('ceph_volume.process.call', lambda x: ('', '', 0)) - vgs = api.VolumeGroups() + vgs = lvm_api.VolumeGroups() vgs._purge() return vgs @@ -40,3 +54,17 @@ def is_root(monkeypatch): is root (or is sudoing to superuser) can continue as-is """ monkeypatch.setattr('os.getuid', lambda: 0) + + +@pytest.fixture +def tmpfile(tmpdir): + """ + Create a temporary file, optionally filling it with contents, returns an + absolute path to the file when called + """ + def generate_file(name='file', contents=''): + path = os.path.join(str(tmpdir), name) + with open(path, 'w') as fp: + fp.write(contents) + return path + return generate_file diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py index 40df77576..ce623aac9 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py @@ -1,10 +1,15 @@ import pytest -from ceph_volume.devices.lvm import activate, api +from ceph_volume.devices.lvm import activate +from ceph_volume.api import lvm as api class Args(object): def __init__(self, **kw): + # default flags + self.bluestore = False + self.filestore = False + self.auto_detect_objectstore = None for k, v in kw.items(): setattr(self, k, v) @@ -20,7 +25,16 @@ class TestActivate(object): volumes.append(FooVolume) monkeypatch.setattr(api, 'Volumes', lambda: volumes) monkeypatch.setattr(activate, 'activate_filestore', capture) - args = Args(osd_id=None, osd_fsid='1234') + args = Args(osd_id=None, osd_fsid='1234', filestore=True) + activate.Activate([]).activate(args) + assert capture.calls[0]['args'][0] == [FooVolume] + + def test_no_osd_id_matches_fsid_bluestore(self, is_root, volumes, monkeypatch, capture): + FooVolume = api.Volume(lv_name='foo', lv_path='/dev/vg/foo', lv_tags="ceph.osd_fsid=1234") + volumes.append(FooVolume) + monkeypatch.setattr(api, 'Volumes', lambda: volumes) + monkeypatch.setattr(activate, 'activate_bluestore', capture) + args = Args(osd_id=None, osd_fsid='1234', bluestore=True) activate.Activate([]).activate(args) assert capture.calls[0]['args'][0] == [FooVolume] @@ -32,3 +46,33 @@ class TestActivate(object): args = Args(osd_id=None, osd_fsid='1234') with pytest.raises(RuntimeError): activate.Activate([]).activate(args) + + +class TestActivateFlags(object): + + def test_default_objectstore(self, capture): + args = ['0', 'asdf-ljh-asdf'] + activation = activate.Activate(args) + activation.activate = capture + activation.main() + parsed_args = capture.calls[0]['args'][0] + assert parsed_args.filestore is False + assert parsed_args.bluestore is True + + def test_uses_filestore(self, capture): + args = ['--filestore', '0', 'asdf-ljh-asdf'] + activation = activate.Activate(args) + activation.activate = capture + activation.main() + parsed_args = capture.calls[0]['args'][0] + assert parsed_args.filestore is True + assert parsed_args.bluestore is False + + def test_uses_bluestore(self, capture): + args = ['--bluestore', '0', 'asdf-ljh-asdf'] + activation = activate.Activate(args) + activation.activate = capture + activation.main() + parsed_args = capture.calls[0]['args'][0] + assert parsed_args.filestore is False + assert parsed_args.bluestore is True diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_listing.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_listing.py new file mode 100644 index 000000000..b780ea2e9 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_listing.py @@ -0,0 +1,176 @@ +import pytest +from ceph_volume.devices import lvm +from ceph_volume.api import lvm as api + + +class TestReadableTag(object): + + def test_dots_get_replaced(self): + result = lvm.listing.readable_tag('ceph.foo') + assert result == 'foo' + + def test_underscores_are_replaced_with_spaces(self): + result = lvm.listing.readable_tag('ceph.long_tag') + assert result == 'long tag' + + +class TestPrettyReport(object): + + def test_is_empty(self, capsys): + lvm.listing.pretty_report({}) + stdout, stderr = capsys.readouterr() + assert stdout == '\n' + + def test_type_and_path_are_reported(self, capsys): + lvm.listing.pretty_report({0: [{'type': 'data', 'path': '/dev/sda1'}]}) + stdout, stderr = capsys.readouterr() + assert '[data] /dev/sda1' in stdout + + def test_osd_id_header_is_reported(self, capsys): + lvm.listing.pretty_report({0: [{'type': 'data', 'path': '/dev/sda1'}]}) + stdout, stderr = capsys.readouterr() + assert '====== osd.0 =======' in stdout + + def test_tags_are_included(self, capsys): + lvm.listing.pretty_report( + {0: [{ + 'type': 'data', + 'path': '/dev/sda1', + 'tags': {'ceph.osd_id': '0'} + }]} + ) + stdout, stderr = capsys.readouterr() + assert 'osd id' in stdout + + +class TestList(object): + + def test_empty_full_json_zero_exit_status(self, is_root, volumes, factory, capsys): + args = factory(format='json', device=None) + lvm.listing.List([]).list(args) + stdout, stderr = capsys.readouterr() + assert stdout == '{}\n' + + def test_empty_device_json_zero_exit_status(self, is_root, volumes, factory, capsys): + args = factory(format='json', device='/dev/sda1') + lvm.listing.List([]).list(args) + stdout, stderr = capsys.readouterr() + assert stdout == '{}\n' + + def test_empty_full_zero_exit_status(self, is_root, volumes, factory): + args = factory(format='pretty', device=None) + with pytest.raises(SystemExit): + lvm.listing.List([]).list(args) + + def test_empty_device_zero_exit_status(self, is_root, volumes, factory): + args = factory(format='pretty', device='/dev/sda1') + with pytest.raises(SystemExit): + lvm.listing.List([]).list(args) + + +class TestFullReport(object): + + def test_no_ceph_lvs(self, volumes, monkeypatch): + # ceph lvs are detected by looking into its tags + osd = api.Volume(lv_name='volume1', lv_path='/dev/VolGroup/lv', lv_tags={}) + volumes.append(osd) + monkeypatch.setattr(lvm.listing.api, 'Volumes', lambda: volumes) + result = lvm.listing.List([]).full_report() + assert result == {} + + def test_ceph_data_lv_reported(self, volumes, monkeypatch): + tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data' + osd = api.Volume( + lv_name='volume1', lv_uuid='y', lv_path='/dev/VolGroup/lv', lv_tags=tags) + volumes.append(osd) + monkeypatch.setattr(lvm.listing.api, 'Volumes', lambda: volumes) + result = lvm.listing.List([]).full_report() + assert result['0'][0]['name'] == 'volume1' + + def test_ceph_journal_lv_reported(self, volumes, monkeypatch): + tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data' + journal_tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=journal' + osd = api.Volume( + lv_name='volume1', lv_uuid='y', lv_path='/dev/VolGroup/lv', lv_tags=tags) + journal = api.Volume( + lv_name='journal', lv_uuid='x', lv_path='/dev/VolGroup/journal', lv_tags=journal_tags) + volumes.append(osd) + volumes.append(journal) + monkeypatch.setattr(lvm.listing.api, 'Volumes', lambda: volumes) + result = lvm.listing.List([]).full_report() + assert result['0'][0]['name'] == 'volume1' + assert result['0'][1]['name'] == 'journal' + + def test_ceph_wal_lv_reported(self, volumes, monkeypatch): + tags = 'ceph.osd_id=0,ceph.wal_uuid=x,ceph.type=data' + wal_tags = 'ceph.osd_id=0,ceph.wal_uuid=x,ceph.type=wal' + osd = api.Volume( + lv_name='volume1', lv_uuid='y', lv_path='/dev/VolGroup/lv', lv_tags=tags) + wal = api.Volume( + lv_name='wal', lv_uuid='x', lv_path='/dev/VolGroup/wal', lv_tags=wal_tags) + volumes.append(osd) + volumes.append(wal) + monkeypatch.setattr(lvm.listing.api, 'Volumes', lambda: volumes) + result = lvm.listing.List([]).full_report() + assert result['0'][0]['name'] == 'volume1' + assert result['0'][1]['name'] == 'wal' + + def test_physical_journal_gets_reported(self, volumes, monkeypatch): + tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data' + osd = api.Volume( + lv_name='volume1', lv_uuid='y', lv_path='/dev/VolGroup/lv', lv_tags=tags) + volumes.append(osd) + monkeypatch.setattr(lvm.listing.api, 'Volumes', lambda: volumes) + monkeypatch.setattr(lvm.listing.disk, 'get_device_from_partuuid', lambda x: '/dev/sda1') + result = lvm.listing.List([]).full_report() + assert result['0'][1]['path'] == '/dev/sda1' + assert result['0'][1]['tags'] == {'PARTUUID': 'x'} + assert result['0'][1]['type'] == 'journal' + + def test_physical_wal_gets_reported(self, volumes, monkeypatch): + tags = 'ceph.osd_id=0,ceph.wal_uuid=x,ceph.type=data' + osd = api.Volume( + lv_name='volume1', lv_uuid='y', lv_path='/dev/VolGroup/lv', lv_tags=tags) + volumes.append(osd) + monkeypatch.setattr(lvm.listing.api, 'Volumes', lambda: volumes) + monkeypatch.setattr(lvm.listing.disk, 'get_device_from_partuuid', lambda x: '/dev/sda1') + result = lvm.listing.List([]).full_report() + assert result['0'][1]['path'] == '/dev/sda1' + assert result['0'][1]['tags'] == {'PARTUUID': 'x'} + assert result['0'][1]['type'] == 'wal' + + +class TestSingleReport(object): + + def test_not_a_ceph_lv(self, volumes, monkeypatch): + # ceph lvs are detected by looking into its tags + lv = api.Volume( + lv_name='lv', vg_name='VolGroup', lv_path='/dev/VolGroup/lv', lv_tags={}) + volumes.append(lv) + monkeypatch.setattr(lvm.listing.api, 'Volumes', lambda: volumes) + result = lvm.listing.List([]).single_report('VolGroup/lv') + assert result == {} + + def test_report_a_ceph_lv(self, volumes, monkeypatch): + # ceph lvs are detected by looking into its tags + tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data' + lv = api.Volume( + lv_name='lv', vg_name='VolGroup', lv_path='/dev/VolGroup/lv', lv_tags=tags) + volumes.append(lv) + monkeypatch.setattr(lvm.listing.api, 'Volumes', lambda: volumes) + result = lvm.listing.List([]).single_report('VolGroup/lv') + assert result['0'][0]['name'] == 'lv' + assert result['0'][0]['lv_tags'] == tags + assert result['0'][0]['path'] == '/dev/VolGroup/lv' + + def test_report_a_ceph_journal_device(self, volumes, monkeypatch): + # ceph lvs are detected by looking into its tags + tags = 'ceph.osd_id=0,ceph.journal_uuid=x,ceph.type=data,ceph.journal_device=/dev/sda1' + lv = api.Volume( + lv_name='lv', vg_name='VolGroup', lv_path='/dev/VolGroup/lv', lv_tags=tags) + volumes.append(lv) + monkeypatch.setattr(lvm.listing.api, 'Volumes', lambda: volumes) + result = lvm.listing.List([]).single_report('/dev/sda1') + assert result['0'][0]['tags'] == {'PARTUUID': 'x'} + assert result['0'][0]['type'] == 'journal' + assert result['0'][0]['path'] == '/dev/sda1' diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_prepare.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_prepare.py index fabae296a..c69394fd6 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_prepare.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_prepare.py @@ -33,8 +33,9 @@ class TestPrepare(object): with pytest.raises(SystemExit): lvm.prepare.Prepare(argv=['--help']).main() stdout, stderr = capsys.readouterr() - assert 'required arguments:' in stdout - assert 'A logical group name or a path' in stdout + assert 'Use the filestore objectstore' in stdout + assert 'Use the bluestore objectstore' in stdout + assert 'A physical device or logical' in stdout class TestGetJournalLV(object): @@ -43,13 +44,13 @@ class TestGetJournalLV(object): def test_no_journal_on_invalid_path(self, monkeypatch, arg): monkeypatch.setattr(lvm.prepare.api, 'get_lv', lambda **kw: False) prepare = lvm.prepare.Prepare([]) - assert prepare.get_journal_lv(arg) is None + assert prepare.get_lv(arg) is None def test_no_journal_lv_found(self, monkeypatch): # patch it with 0 so we know we are getting to get_lv monkeypatch.setattr(lvm.prepare.api, 'get_lv', lambda **kw: 0) prepare = lvm.prepare.Prepare([]) - assert prepare.get_journal_lv('vg/lv') == 0 + assert prepare.get_lv('vg/lv') == 0 class TestActivate(object): diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/simple/test_activate.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/simple/test_activate.py new file mode 100644 index 000000000..bae3276a9 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/simple/test_activate.py @@ -0,0 +1,23 @@ +import os +import pytest +from ceph_volume.devices.simple import activate + + +class TestActivate(object): + + def test_no_data_uuid(self, factory, tmpfile, is_root, monkeypatch, capture): + json_config = tmpfile(contents='{}') + args = factory(osd_id='0', osd_fsid='1234', json_config=json_config) + with pytest.raises(RuntimeError): + activate.Activate([]).activate(args) + + def test_invalid_json_path(self): + os.environ['CEPH_VOLUME_SIMPLE_JSON_DIR'] = '/non/existing/path' + with pytest.raises(RuntimeError) as error: + activate.Activate(['1', 'asdf']).main() + assert 'RuntimeError: Expected JSON config path not found' in str(error) + + def test_main_spits_help_with_no_arguments(self, capsys): + activate.Activate([]).main() + stdout, stderr = capsys.readouterr() + assert 'Activate OSDs by mounting devices previously configured' in stdout diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/simple/test_scan.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/simple/test_scan.py new file mode 100644 index 000000000..d68fe63cb --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/simple/test_scan.py @@ -0,0 +1,52 @@ +import os +import pytest +from ceph_volume.devices.simple import scan + + +class TestScan(object): + + def test_main_spits_help_with_no_arguments(self, capsys): + scan.Scan([]).main() + stdout, stderr = capsys.readouterr() + assert 'Scan an OSD directory for files' in stdout + + +class TestGetContents(object): + + def test_multiple_lines_are_left_as_is(self, tmpfile): + magic_file = tmpfile(contents='first\nsecond\n') + scanner = scan.Scan([]) + assert scanner.get_contents(magic_file) == 'first\nsecond\n' + + def test_extra_whitespace_gets_removed(self, tmpfile): + magic_file = tmpfile(contents='first ') + scanner = scan.Scan([]) + assert scanner.get_contents(magic_file) == 'first' + + def test_single_newline_values_are_trimmed(self, tmpfile): + magic_file = tmpfile(contents='first\n') + scanner = scan.Scan([]) + assert scanner.get_contents(magic_file) == 'first' + + +class TestEtcPath(object): + + def test_directory_is_valid(self, tmpdir): + path = str(tmpdir) + scanner = scan.Scan([]) + scanner._etc_path = path + assert scanner.etc_path == path + + def test_directory_does_not_exist_gets_created(self, tmpdir): + path = os.path.join(str(tmpdir), 'subdir') + scanner = scan.Scan([]) + scanner._etc_path = path + assert scanner.etc_path == path + assert os.path.isdir(path) + + def test_complains_when_file(self, tmpfile): + path = tmpfile() + scanner = scan.Scan([]) + scanner._etc_path = path + with pytest.raises(RuntimeError): + scanner.etc_path diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/simple/test_trigger.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/simple/test_trigger.py new file mode 100644 index 000000000..d3220f2b0 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/simple/test_trigger.py @@ -0,0 +1,45 @@ +import pytest +from ceph_volume import exceptions +from ceph_volume.devices.simple import trigger + + +class TestParseOSDid(object): + + def test_no_id_found_if_no_digit(self): + with pytest.raises(exceptions.SuffixParsingError): + trigger.parse_osd_id('asdlj-ljahsdfaslkjhdfa') + + def test_no_id_found(self): + with pytest.raises(exceptions.SuffixParsingError): + trigger.parse_osd_id('ljahsdfaslkjhdfa') + + def test_id_found(self): + result = trigger.parse_osd_id('1-ljahsdfaslkjhdfa') + assert result == '1' + + +class TestParseOSDUUID(object): + + def test_uuid_is_parsed(self): + result = trigger.parse_osd_uuid('1-asdf-ljkh-asdf-ljkh-asdf') + assert result == 'asdf-ljkh-asdf-ljkh-asdf' + + def test_uuid_is_parsed_longer_sha1(self): + result = trigger.parse_osd_uuid('1-foo-bar-asdf-ljkh-asdf-ljkh-asdf') + assert result == 'foo-bar-asdf-ljkh-asdf-ljkh-asdf' + + def test_uuid_is_not_found(self): + with pytest.raises(exceptions.SuffixParsingError): + trigger.parse_osd_uuid('ljahsdfaslkjhdfa') + + def test_uuid_is_not_found_missing_id(self): + with pytest.raises(exceptions.SuffixParsingError): + trigger.parse_osd_uuid('ljahs-dfa-slkjhdfa-foo') + + def test_robust_double_id_in_uuid(self): + # it is possible to have the id in the SHA1, this should + # be fine parsing that + result = trigger.parse_osd_uuid("1-abc959fd-1ec9-4864-b141-3154f9b9f8ed") + assert result == 'abc959fd-1ec9-4864-b141-3154f9b9f8ed' + + diff --git a/ceph/src/ceph-volume/ceph_volume/tests/devices/test_zap.py b/ceph/src/ceph-volume/ceph_volume/tests/devices/test_zap.py new file mode 100644 index 000000000..bc26e33da --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/devices/test_zap.py @@ -0,0 +1,17 @@ +import pytest +from ceph_volume.devices import lvm + + +class TestZap(object): + + def test_main_spits_help_with_no_arguments(self, capsys): + lvm.zap.Zap([]).main() + stdout, stderr = capsys.readouterr() + assert 'Zaps the given logical volume or partition' in stdout + + def test_main_shows_full_help(self, capsys): + with pytest.raises(SystemExit): + lvm.zap.Zap(argv=['--help']).main() + stdout, stderr = capsys.readouterr() + assert 'optional arguments' in stdout + assert 'positional arguments' in stdout diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/centos7/create/Vagrantfile b/ceph/src/ceph-volume/ceph_volume/tests/functional/centos7/create/Vagrantfile deleted file mode 120000 index 2572fa2c9..000000000 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/centos7/create/Vagrantfile +++ /dev/null @@ -1 +0,0 @@ -../../Vagrantfile \ No newline at end of file diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/Vagrantfile b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/Vagrantfile new file mode 120000 index 000000000..16076e424 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/Vagrantfile @@ -0,0 +1 @@ +../../../../Vagrantfile \ No newline at end of file diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/group_vars/all new file mode 100644 index 000000000..17e9044e1 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/group_vars/all @@ -0,0 +1,28 @@ +--- + +ceph_dev: True +cluster: ceph +public_network: "192.168.3.0/24" +cluster_network: "192.168.4.0/24" +monitor_interface: eth1 +journal_size: 100 +osd_objectstore: "bluestore" +osd_scenario: lvm +ceph_origin: 'repository' +ceph_repository: 'dev' +copy_admin_key: true +lvm_volumes: + - data: data-lv1 + data_vg: test_group + - data: data-lv2 + data_vg: test_group + db: journal1 + db_vg: journals + - data: /dev/sdd1 +os_tuning_params: + - { name: kernel.pid_max, value: 4194303 } + - { name: fs.file-max, value: 26234859 } +ceph_conf_overrides: + global: + osd_pool_default_pg_num: 8 + osd_pool_default_size: 1 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/centos7/create/hosts b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/hosts similarity index 100% rename from ceph/src/ceph-volume/ceph_volume/tests/functional/centos7/create/hosts rename to ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/hosts diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/setup.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/setup.yml new file mode 120000 index 000000000..1c1a3ce8d --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/setup.yml @@ -0,0 +1 @@ +../../../playbooks/setup_partitions.yml \ No newline at end of file diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/centos7/create/vagrant_variables.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/vagrant_variables.yml similarity index 100% rename from ceph/src/ceph-volume/ceph_volume/tests/functional/centos7/create/vagrant_variables.yml rename to ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/bluestore/create/vagrant_variables.yml diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/Vagrantfile b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/Vagrantfile new file mode 120000 index 000000000..16076e424 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/Vagrantfile @@ -0,0 +1 @@ +../../../../Vagrantfile \ No newline at end of file diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/centos7/create/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/group_vars/all similarity index 94% rename from ceph/src/ceph-volume/ceph_volume/tests/functional/centos7/create/group_vars/all rename to ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/group_vars/all index e7c1f7230..e7ff18ed1 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/centos7/create/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/group_vars/all @@ -20,6 +20,8 @@ lvm_volumes: journal: journal1 data_vg: test_group journal_vg: journals + - data: /dev/sdd1 + journal: /dev/sdd2 os_tuning_params: - { name: kernel.pid_max, value: 4194303 } - { name: fs.file-max, value: 26234859 } diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/xenial/create/hosts b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/hosts similarity index 100% rename from ceph/src/ceph-volume/ceph_volume/tests/functional/xenial/create/hosts rename to ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/hosts diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/setup.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/setup.yml new file mode 120000 index 000000000..1c1a3ce8d --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/setup.yml @@ -0,0 +1 @@ +../../../playbooks/setup_partitions.yml \ No newline at end of file diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/vagrant_variables.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/vagrant_variables.yml new file mode 100644 index 000000000..7d1a4449a --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/centos7/filestore/create/vagrant_variables.yml @@ -0,0 +1,56 @@ +--- + +# DEFINE THE NUMBER OF VMS TO RUN +mon_vms: 1 +osd_vms: 1 +mds_vms: 0 +rgw_vms: 0 +nfs_vms: 0 +rbd_mirror_vms: 0 +client_vms: 0 +iscsi_gw_vms: 0 +mgr_vms: 0 + +# SUBNETS TO USE FOR THE VMS +public_subnet: 192.168.3 +cluster_subnet: 192.168.4 + +# MEMORY +# set 1024 for CentOS +memory: 512 + +# Ethernet interface name +# use eth1 for libvirt and ubuntu precise, enp0s8 for CentOS and ubuntu xenial +eth: 'eth1' + + +# VAGRANT BOX +# Ceph boxes are *strongly* suggested. They are under better control and will +# not get updated frequently unless required for build systems. These are (for +# now): +# +# * ceph/ubuntu-xenial +# +# Ubuntu: ceph/ubuntu-xenial bento/ubuntu-16.04 or ubuntu/trusty64 or ubuntu/wily64 +# CentOS: bento/centos-7.1 or puppetlabs/centos-7.0-64-puppet +# libvirt CentOS: centos/7 +# parallels Ubuntu: parallels/ubuntu-14.04 +# Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller' +# For more boxes have a look at: +# - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q= +# - https://download.gluster.org/pub/gluster/purpleidea/vagrant/ +vagrant_box: centos/7 +#ssh_private_key_path: "~/.ssh/id_rsa" +# The sync directory changes based on vagrant box +# Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant +#vagrant_sync_dir: /home/vagrant/sync +#vagrant_sync_dir: / +# Disables synced folder creation. Not needed for testing, will skip mounting +# the vagrant directory on the remote box regardless of the provider. +vagrant_disable_synced_folder: true +# VAGRANT URL +# This is a URL to download an image from an alternate location. vagrant_box +# above should be set to the filename of the image. +# Fedora virtualbox: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box +# Fedora libvirt: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-libvirt.box +# vagrant_box_url: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/setup_partitions.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/setup_partitions.yml new file mode 100644 index 000000000..37a48949b --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/setup_partitions.yml @@ -0,0 +1,27 @@ +--- + +- hosts: osds + gather_facts: false + become: yes + tasks: + + - name: partition /dev/sdd for lvm data usage + parted: + device: /dev/sdd + number: 1 + part_start: 0% + part_end: 50% + unit: '%' + label: gpt + state: present + + - name: partition /dev/sdd lvm journals + parted: + device: /dev/sdd + number: 2 + part_start: 50% + part_end: 100% + unit: '%' + state: present + label: gpt + diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini new file mode 100644 index 000000000..797138f7d --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini @@ -0,0 +1,59 @@ +[tox] +envlist = {centos7,xenial}-{filestore,bluestore}-{create,prepare_activate} +skipsdist = True + +[testenv] +whitelist_externals = + vagrant + bash + git +passenv=* +setenv= + ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config + ANSIBLE_STDOUT_CALLBACK = debug + ANSIBLE_RETRY_FILES_ENABLED = False + VAGRANT_CWD = {changedir} + CEPH_VOLUME_DEBUG = 1 +deps= + ansible==2.4.1 + testinfra==1.7.1 + pytest-xdist +changedir= + centos7-filestore-create: {toxinidir}/centos7/filestore/create + centos7-bluestore-create: {toxinidir}/centos7/bluestore/create + xenial-filestore-create: {toxinidir}/xenial/filestore/create + xenial-bluestore-create: {toxinidir}/xenial/bluestore/create + # TODO: these are placeholders for now, eventually we want to + # test the prepare/activate workflow of ceph-volume as well + xenial-filestore-prepare_activate: {toxinidir}/xenial/filestore/prepare_activate + xenial-bluestore-prepare_activate: {toxinidir}/xenial/bluestore/prepare_activate + centos7-filestore-prepare_activate: {toxinidir}/xenial/filestore/prepare_activate + centos7-bluestore-prepare_activate: {toxinidir}/xenial/bluestore/prepare_activate +commands= + git clone -b {env:CEPH_ANSIBLE_BRANCH:master} --single-branch https://github.com/ceph/ceph-ansible.git {envdir}/tmp/ceph-ansible + + vagrant up --no-provision {posargs:--provider=virtualbox} + bash {toxinidir}/../scripts/generate_ssh_config.sh {changedir} + + # create logical volumes to test with on the vms + ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/lvm_setup.yml + + # ad-hoc/local test setup for lvm + ansible-playbook -vv -i {changedir}/hosts {changedir}/setup.yml + + # use ceph-ansible to deploy a ceph cluster on the vms + ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/site.yml.sample --extra-vars "fetch_directory={changedir}/fetch ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest}" + + # prepare nodes for testing with testinfra + ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/setup.yml + + # test cluster state using ceph-ansible tests + testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/tests + + # reboot all vms + vagrant reload --no-provision + + # retest to ensure cluster came back up correctly after rebooting + testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/tests + + vagrant destroy --force diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/Vagrantfile b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/Vagrantfile new file mode 120000 index 000000000..16076e424 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/Vagrantfile @@ -0,0 +1 @@ +../../../../Vagrantfile \ No newline at end of file diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/group_vars/all new file mode 100644 index 000000000..17e9044e1 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/group_vars/all @@ -0,0 +1,28 @@ +--- + +ceph_dev: True +cluster: ceph +public_network: "192.168.3.0/24" +cluster_network: "192.168.4.0/24" +monitor_interface: eth1 +journal_size: 100 +osd_objectstore: "bluestore" +osd_scenario: lvm +ceph_origin: 'repository' +ceph_repository: 'dev' +copy_admin_key: true +lvm_volumes: + - data: data-lv1 + data_vg: test_group + - data: data-lv2 + data_vg: test_group + db: journal1 + db_vg: journals + - data: /dev/sdd1 +os_tuning_params: + - { name: kernel.pid_max, value: 4194303 } + - { name: fs.file-max, value: 26234859 } +ceph_conf_overrides: + global: + osd_pool_default_pg_num: 8 + osd_pool_default_size: 1 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/hosts b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/hosts new file mode 100644 index 000000000..f6a265ab3 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/hosts @@ -0,0 +1,5 @@ +[mons] +mon0 + +[osds] +osd0 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/setup.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/setup.yml new file mode 120000 index 000000000..1c1a3ce8d --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/setup.yml @@ -0,0 +1 @@ +../../../playbooks/setup_partitions.yml \ No newline at end of file diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/vagrant_variables.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/vagrant_variables.yml new file mode 100644 index 000000000..7252344dd --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/bluestore/create/vagrant_variables.yml @@ -0,0 +1,56 @@ +--- + +# DEFINE THE NUMBER OF VMS TO RUN +mon_vms: 1 +osd_vms: 1 +mds_vms: 0 +rgw_vms: 0 +nfs_vms: 0 +rbd_mirror_vms: 0 +client_vms: 0 +iscsi_gw_vms: 0 +mgr_vms: 0 + +# SUBNETS TO USE FOR THE VMS +public_subnet: 192.168.3 +cluster_subnet: 192.168.4 + +# MEMORY +# set 1024 for CentOS +memory: 512 + +# Ethernet interface name +# use eth1 for libvirt and ubuntu precise, enp0s8 for CentOS and ubuntu xenial +eth: 'eth1' + + +# VAGRANT BOX +# Ceph boxes are *strongly* suggested. They are under better control and will +# not get updated frequently unless required for build systems. These are (for +# now): +# +# * ceph/ubuntu-xenial +# +# Ubuntu: ceph/ubuntu-xenial bento/ubuntu-16.04 or ubuntu/trusty64 or ubuntu/wily64 +# CentOS: bento/centos-7.1 or puppetlabs/centos-7.0-64-puppet +# libvirt CentOS: centos/7 +# parallels Ubuntu: parallels/ubuntu-14.04 +# Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller' +# For more boxes have a look at: +# - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q= +# - https://download.gluster.org/pub/gluster/purpleidea/vagrant/ +vagrant_box: ceph/ubuntu-xenial +#ssh_private_key_path: "~/.ssh/id_rsa" +# The sync directory changes based on vagrant box +# Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant +#vagrant_sync_dir: /home/vagrant/sync +#vagrant_sync_dir: / +# Disables synced folder creation. Not needed for testing, will skip mounting +# the vagrant directory on the remote box regardless of the provider. +vagrant_disable_synced_folder: true +# VAGRANT URL +# This is a URL to download an image from an alternate location. vagrant_box +# above should be set to the filename of the image. +# Fedora virtualbox: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box +# Fedora libvirt: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-libvirt.box +# vagrant_box_url: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/Vagrantfile b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/Vagrantfile new file mode 120000 index 000000000..16076e424 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/Vagrantfile @@ -0,0 +1 @@ +../../../../Vagrantfile \ No newline at end of file diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/xenial/create/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/group_vars/all similarity index 94% rename from ceph/src/ceph-volume/ceph_volume/tests/functional/xenial/create/group_vars/all rename to ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/group_vars/all index e7c1f7230..e7ff18ed1 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/xenial/create/group_vars/all +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/group_vars/all @@ -20,6 +20,8 @@ lvm_volumes: journal: journal1 data_vg: test_group journal_vg: journals + - data: /dev/sdd1 + journal: /dev/sdd2 os_tuning_params: - { name: kernel.pid_max, value: 4194303 } - { name: fs.file-max, value: 26234859 } diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/hosts b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/hosts new file mode 100644 index 000000000..f6a265ab3 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/hosts @@ -0,0 +1,5 @@ +[mons] +mon0 + +[osds] +osd0 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/setup.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/setup.yml new file mode 120000 index 000000000..1c1a3ce8d --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/setup.yml @@ -0,0 +1 @@ +../../../playbooks/setup_partitions.yml \ No newline at end of file diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/xenial/create/vagrant_variables.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/vagrant_variables.yml similarity index 100% rename from ceph/src/ceph-volume/ceph_volume/tests/functional/xenial/create/vagrant_variables.yml rename to ceph/src/ceph-volume/ceph_volume/tests/functional/lvm/xenial/filestore/create/vagrant_variables.yml diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/Vagrantfile b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/Vagrantfile new file mode 120000 index 000000000..16076e424 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/Vagrantfile @@ -0,0 +1 @@ +../../../../Vagrantfile \ No newline at end of file diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/group_vars/all new file mode 100644 index 000000000..560c8b03b --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/group_vars/all @@ -0,0 +1,19 @@ +--- + +ceph_dev: True +cluster: test +public_network: "192.168.1.0/24" +cluster_network: "192.168.2.0/24" +monitor_interface: eth1 +journal_size: 100 +osd_objectstore: "bluestore" +ceph_origin: 'repository' +ceph_repository: 'dev' +copy_admin_key: true +os_tuning_params: + - { name: kernel.pid_max, value: 4194303 } + - { name: fs.file-max, value: 26234859 } +ceph_conf_overrides: + global: + osd_pool_default_pg_num: 8 + osd_pool_default_size: 1 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/host_vars/osd0.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/host_vars/osd0.yml new file mode 100644 index 000000000..2e1c7ee9e --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/host_vars/osd0.yml @@ -0,0 +1,7 @@ +--- + +devices: + - '/dev/sdb' +dedicated_devices: + - '/dev/sdc' +osd_scenario: "non-collocated" diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/host_vars/osd1.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/host_vars/osd1.yml new file mode 100644 index 000000000..7e90071c9 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/host_vars/osd1.yml @@ -0,0 +1,6 @@ +--- + +devices: + - '/dev/sdb' + - '/dev/sdc' +osd_scenario: "collocated" diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/hosts b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/hosts new file mode 100644 index 000000000..e0c08b946 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/hosts @@ -0,0 +1,9 @@ +[mons] +mon0 monitor_interface=eth1 + +[osds] +osd0 +osd1 + +[mgrs] +mon0 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/test.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/test.yml new file mode 100644 index 000000000..24e2c0353 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/test.yml @@ -0,0 +1,31 @@ +--- + +- hosts: osds + become: yes + tasks: + + - name: list all OSD directories + find: + paths: /var/lib/ceph/osd + file_type: directory + register: osd_paths + + - name: scan all OSD directories + command: "ceph-volume --cluster={{ cluster }} simple scan {{ item.path }}" + environment: + CEPH_VOLUME_DEBUG: 1 + with_items: + - "{{ osd_paths.files }}" + + - name: list all OSD JSON files + find: + paths: /etc/ceph/osd + file_type: file + register: osd_configs + + - name: activate all scanned OSDs + command: "ceph-volume --cluster={{ cluster }} simple activate --file {{ item.path }}" + environment: + CEPH_VOLUME_DEBUG: 1 + with_items: + - "{{ osd_configs.files }}" diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/vagrant_variables.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/vagrant_variables.yml new file mode 100644 index 000000000..63700c3c9 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/vagrant_variables.yml @@ -0,0 +1,73 @@ +--- + +# DEPLOY CONTAINERIZED DAEMONS +docker: false + +# DEFINE THE NUMBER OF VMS TO RUN +mon_vms: 1 +osd_vms: 2 +mds_vms: 0 +rgw_vms: 0 +nfs_vms: 0 +rbd_mirror_vms: 0 +client_vms: 0 +iscsi_gw_vms: 0 +mgr_vms: 0 + + +# INSTALL SOURCE OF CEPH +# valid values are 'stable' and 'dev' +ceph_install_source: stable + +# SUBNETS TO USE FOR THE VMS +public_subnet: 192.168.1 +cluster_subnet: 192.168.2 + +# MEMORY +# set 1024 for CentOS +memory: 512 + +# Ethernet interface name +# use eth1 for libvirt and ubuntu precise, enp0s8 for CentOS and ubuntu xenial +eth: 'eth1' + +# Disks +# For libvirt use disks: "[ '/dev/vdb', '/dev/vdc' ]" +# For CentOS7 use disks: "[ '/dev/sda', '/dev/sdb' ]" +disks: "[ '/dev/sdb', '/dev/sdc' ]" + +# VAGRANT BOX +# Ceph boxes are *strongly* suggested. They are under better control and will +# not get updated frequently unless required for build systems. These are (for +# now): +# +# * ceph/ubuntu-xenial +# +# Ubuntu: ceph/ubuntu-xenial bento/ubuntu-16.04 or ubuntu/trusty64 or ubuntu/wily64 +# CentOS: bento/centos-7.1 or puppetlabs/centos-7.0-64-puppet +# libvirt CentOS: centos/7 +# parallels Ubuntu: parallels/ubuntu-14.04 +# Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller' +# For more boxes have a look at: +# - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q= +# - https://download.gluster.org/pub/gluster/purpleidea/vagrant/ +vagrant_box: centos/7 +#ssh_private_key_path: "~/.ssh/id_rsa" +# The sync directory changes based on vagrant box +# Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant +#vagrant_sync_dir: /home/vagrant/sync +#vagrant_sync_dir: / +# Disables synced folder creation. Not needed for testing, will skip mounting +# the vagrant directory on the remote box regardless of the provider. +vagrant_disable_synced_folder: true +# VAGRANT URL +# This is a URL to download an image from an alternate location. vagrant_box +# above should be set to the filename of the image. +# Fedora virtualbox: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box +# Fedora libvirt: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-libvirt.box +# vagrant_box_url: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box + +os_tuning_params: + - { name: kernel.pid_max, value: 4194303 } + - { name: fs.file-max, value: 26234859 } + diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/Vagrantfile b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/Vagrantfile new file mode 120000 index 000000000..16076e424 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/Vagrantfile @@ -0,0 +1 @@ +../../../../Vagrantfile \ No newline at end of file diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/group_vars/all new file mode 100644 index 000000000..8902bdda3 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/group_vars/all @@ -0,0 +1,19 @@ +--- + +ceph_dev: True +cluster: test +public_network: "192.168.1.0/24" +cluster_network: "192.168.2.0/24" +monitor_interface: eth1 +journal_size: 100 +osd_objectstore: "filestore" +ceph_origin: 'repository' +ceph_repository: 'dev' +copy_admin_key: true +os_tuning_params: + - { name: kernel.pid_max, value: 4194303 } + - { name: fs.file-max, value: 26234859 } +ceph_conf_overrides: + global: + osd_pool_default_pg_num: 8 + osd_pool_default_size: 1 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/host_vars/osd0.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/host_vars/osd0.yml new file mode 100644 index 000000000..2e1c7ee9e --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/host_vars/osd0.yml @@ -0,0 +1,7 @@ +--- + +devices: + - '/dev/sdb' +dedicated_devices: + - '/dev/sdc' +osd_scenario: "non-collocated" diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/host_vars/osd1.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/host_vars/osd1.yml new file mode 100644 index 000000000..7e90071c9 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/host_vars/osd1.yml @@ -0,0 +1,6 @@ +--- + +devices: + - '/dev/sdb' + - '/dev/sdc' +osd_scenario: "collocated" diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/hosts b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/hosts new file mode 100644 index 000000000..e0c08b946 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/hosts @@ -0,0 +1,9 @@ +[mons] +mon0 monitor_interface=eth1 + +[osds] +osd0 +osd1 + +[mgrs] +mon0 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/test.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/test.yml new file mode 100644 index 000000000..24e2c0353 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/test.yml @@ -0,0 +1,31 @@ +--- + +- hosts: osds + become: yes + tasks: + + - name: list all OSD directories + find: + paths: /var/lib/ceph/osd + file_type: directory + register: osd_paths + + - name: scan all OSD directories + command: "ceph-volume --cluster={{ cluster }} simple scan {{ item.path }}" + environment: + CEPH_VOLUME_DEBUG: 1 + with_items: + - "{{ osd_paths.files }}" + + - name: list all OSD JSON files + find: + paths: /etc/ceph/osd + file_type: file + register: osd_configs + + - name: activate all scanned OSDs + command: "ceph-volume --cluster={{ cluster }} simple activate --file {{ item.path }}" + environment: + CEPH_VOLUME_DEBUG: 1 + with_items: + - "{{ osd_configs.files }}" diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/vagrant_variables.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/vagrant_variables.yml new file mode 100644 index 000000000..63700c3c9 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/filestore/activate/vagrant_variables.yml @@ -0,0 +1,73 @@ +--- + +# DEPLOY CONTAINERIZED DAEMONS +docker: false + +# DEFINE THE NUMBER OF VMS TO RUN +mon_vms: 1 +osd_vms: 2 +mds_vms: 0 +rgw_vms: 0 +nfs_vms: 0 +rbd_mirror_vms: 0 +client_vms: 0 +iscsi_gw_vms: 0 +mgr_vms: 0 + + +# INSTALL SOURCE OF CEPH +# valid values are 'stable' and 'dev' +ceph_install_source: stable + +# SUBNETS TO USE FOR THE VMS +public_subnet: 192.168.1 +cluster_subnet: 192.168.2 + +# MEMORY +# set 1024 for CentOS +memory: 512 + +# Ethernet interface name +# use eth1 for libvirt and ubuntu precise, enp0s8 for CentOS and ubuntu xenial +eth: 'eth1' + +# Disks +# For libvirt use disks: "[ '/dev/vdb', '/dev/vdc' ]" +# For CentOS7 use disks: "[ '/dev/sda', '/dev/sdb' ]" +disks: "[ '/dev/sdb', '/dev/sdc' ]" + +# VAGRANT BOX +# Ceph boxes are *strongly* suggested. They are under better control and will +# not get updated frequently unless required for build systems. These are (for +# now): +# +# * ceph/ubuntu-xenial +# +# Ubuntu: ceph/ubuntu-xenial bento/ubuntu-16.04 or ubuntu/trusty64 or ubuntu/wily64 +# CentOS: bento/centos-7.1 or puppetlabs/centos-7.0-64-puppet +# libvirt CentOS: centos/7 +# parallels Ubuntu: parallels/ubuntu-14.04 +# Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller' +# For more boxes have a look at: +# - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q= +# - https://download.gluster.org/pub/gluster/purpleidea/vagrant/ +vagrant_box: centos/7 +#ssh_private_key_path: "~/.ssh/id_rsa" +# The sync directory changes based on vagrant box +# Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant +#vagrant_sync_dir: /home/vagrant/sync +#vagrant_sync_dir: / +# Disables synced folder creation. Not needed for testing, will skip mounting +# the vagrant directory on the remote box regardless of the provider. +vagrant_disable_synced_folder: true +# VAGRANT URL +# This is a URL to download an image from an alternate location. vagrant_box +# above should be set to the filename of the image. +# Fedora virtualbox: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box +# Fedora libvirt: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-libvirt.box +# vagrant_box_url: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box + +os_tuning_params: + - { name: kernel.pid_max, value: 4194303 } + - { name: fs.file-max, value: 26234859 } + diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/tox.ini b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini similarity index 68% rename from ceph/src/ceph-volume/ceph_volume/tests/functional/tox.ini rename to ceph/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini index 6e0dfbf2d..0d2e68adc 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/tox.ini +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {centos7,xenial}-{create,prepare_activate} +envlist = {centos7,xenial}-{filestore,bluestore}-{activate} skipsdist = True [testenv] @@ -15,24 +15,19 @@ setenv= VAGRANT_CWD = {changedir} CEPH_VOLUME_DEBUG = 1 deps= - ansible==2.3.1 - testinfra==1.6.0 + ansible==2.4.1 + testinfra==1.7.1 pytest-xdist changedir= - centos7-create: {toxinidir}/centos7/create - xenial-create: {toxinidir}/xenial/create - # TODO: these are placeholders for now, eventually we want to - # test the prepare/activate workflow of ceph-volume as well - xenial-prepare_activate: {toxinidir}/xenial/prepare_activate - centos7-prepare_activate: {toxinidir}/xenial/prepare_activate + centos7-filestore-activate: {toxinidir}/centos7/filestore/activate + centos7-bluestore-activate: {toxinidir}/centos7/bluestore/activate + xenial-filestore-activate: {toxinidir}/xenial/filestore/activate + xenial-bluestore-activate: {toxinidir}/xenial/bluestore/activate commands= git clone -b {env:CEPH_ANSIBLE_BRANCH:master} --single-branch https://github.com/ceph/ceph-ansible.git {envdir}/tmp/ceph-ansible vagrant up --no-provision {posargs:--provider=virtualbox} - bash {toxinidir}/scripts/generate_ssh_config.sh {changedir} - - # create logical volumes to test with on the vms - ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/lvm_setup.yml + bash {toxinidir}/../scripts/generate_ssh_config.sh {changedir} # use ceph-ansible to deploy a ceph cluster on the vms ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/site.yml.sample --extra-vars "fetch_directory={changedir}/fetch ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest}" @@ -43,6 +38,9 @@ commands= # test cluster state using ceph-ansible tests testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/tests + # make ceph-volume simple take over all the OSDs that got deployed, disabling ceph-disk + ansible-playbook -vv -i {changedir}/hosts {changedir}/test.yml + # reboot all vms vagrant reload --no-provision diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/Vagrantfile b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/Vagrantfile new file mode 120000 index 000000000..16076e424 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/Vagrantfile @@ -0,0 +1 @@ +../../../../Vagrantfile \ No newline at end of file diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/group_vars/all new file mode 100644 index 000000000..560c8b03b --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/group_vars/all @@ -0,0 +1,19 @@ +--- + +ceph_dev: True +cluster: test +public_network: "192.168.1.0/24" +cluster_network: "192.168.2.0/24" +monitor_interface: eth1 +journal_size: 100 +osd_objectstore: "bluestore" +ceph_origin: 'repository' +ceph_repository: 'dev' +copy_admin_key: true +os_tuning_params: + - { name: kernel.pid_max, value: 4194303 } + - { name: fs.file-max, value: 26234859 } +ceph_conf_overrides: + global: + osd_pool_default_pg_num: 8 + osd_pool_default_size: 1 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/host_vars/osd0.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/host_vars/osd0.yml new file mode 100644 index 000000000..2e1c7ee9e --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/host_vars/osd0.yml @@ -0,0 +1,7 @@ +--- + +devices: + - '/dev/sdb' +dedicated_devices: + - '/dev/sdc' +osd_scenario: "non-collocated" diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/host_vars/osd1.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/host_vars/osd1.yml new file mode 100644 index 000000000..7e90071c9 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/host_vars/osd1.yml @@ -0,0 +1,6 @@ +--- + +devices: + - '/dev/sdb' + - '/dev/sdc' +osd_scenario: "collocated" diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/hosts b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/hosts new file mode 100644 index 000000000..e0c08b946 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/hosts @@ -0,0 +1,9 @@ +[mons] +mon0 monitor_interface=eth1 + +[osds] +osd0 +osd1 + +[mgrs] +mon0 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/test.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/test.yml new file mode 100644 index 000000000..24e2c0353 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/test.yml @@ -0,0 +1,31 @@ +--- + +- hosts: osds + become: yes + tasks: + + - name: list all OSD directories + find: + paths: /var/lib/ceph/osd + file_type: directory + register: osd_paths + + - name: scan all OSD directories + command: "ceph-volume --cluster={{ cluster }} simple scan {{ item.path }}" + environment: + CEPH_VOLUME_DEBUG: 1 + with_items: + - "{{ osd_paths.files }}" + + - name: list all OSD JSON files + find: + paths: /etc/ceph/osd + file_type: file + register: osd_configs + + - name: activate all scanned OSDs + command: "ceph-volume --cluster={{ cluster }} simple activate --file {{ item.path }}" + environment: + CEPH_VOLUME_DEBUG: 1 + with_items: + - "{{ osd_configs.files }}" diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/vagrant_variables.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/vagrant_variables.yml new file mode 100644 index 000000000..b4aa759ab --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/bluestore/activate/vagrant_variables.yml @@ -0,0 +1,73 @@ +--- + +# DEPLOY CONTAINERIZED DAEMONS +docker: false + +# DEFINE THE NUMBER OF VMS TO RUN +mon_vms: 1 +osd_vms: 2 +mds_vms: 0 +rgw_vms: 0 +nfs_vms: 0 +rbd_mirror_vms: 0 +client_vms: 0 +iscsi_gw_vms: 0 +mgr_vms: 0 + + +# INSTALL SOURCE OF CEPH +# valid values are 'stable' and 'dev' +ceph_install_source: stable + +# SUBNETS TO USE FOR THE VMS +public_subnet: 192.168.1 +cluster_subnet: 192.168.2 + +# MEMORY +# set 1024 for CentOS +memory: 512 + +# Ethernet interface name +# use eth1 for libvirt and ubuntu precise, enp0s8 for CentOS and ubuntu xenial +eth: 'eth1' + +# Disks +# For libvirt use disks: "[ '/dev/vdb', '/dev/vdc' ]" +# For CentOS7 use disks: "[ '/dev/sda', '/dev/sdb' ]" +disks: "[ '/dev/sdb', '/dev/sdc' ]" + +# VAGRANT BOX +# Ceph boxes are *strongly* suggested. They are under better control and will +# not get updated frequently unless required for build systems. These are (for +# now): +# +# * ceph/ubuntu-xenial +# +# Ubuntu: ceph/ubuntu-xenial bento/ubuntu-16.04 or ubuntu/trusty64 or ubuntu/wily64 +# CentOS: bento/centos-7.1 or puppetlabs/centos-7.0-64-puppet +# libvirt CentOS: centos/7 +# parallels Ubuntu: parallels/ubuntu-14.04 +# Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller' +# For more boxes have a look at: +# - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q= +# - https://download.gluster.org/pub/gluster/purpleidea/vagrant/ +vagrant_box: ceph/ubuntu-xenial +#ssh_private_key_path: "~/.ssh/id_rsa" +# The sync directory changes based on vagrant box +# Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant +#vagrant_sync_dir: /home/vagrant/sync +#vagrant_sync_dir: / +# Disables synced folder creation. Not needed for testing, will skip mounting +# the vagrant directory on the remote box regardless of the provider. +vagrant_disable_synced_folder: true +# VAGRANT URL +# This is a URL to download an image from an alternate location. vagrant_box +# above should be set to the filename of the image. +# Fedora virtualbox: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box +# Fedora libvirt: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-libvirt.box +# vagrant_box_url: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box + +os_tuning_params: + - { name: kernel.pid_max, value: 4194303 } + - { name: fs.file-max, value: 26234859 } + diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/Vagrantfile b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/Vagrantfile new file mode 120000 index 000000000..16076e424 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/Vagrantfile @@ -0,0 +1 @@ +../../../../Vagrantfile \ No newline at end of file diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/group_vars/all b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/group_vars/all new file mode 100644 index 000000000..8902bdda3 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/group_vars/all @@ -0,0 +1,19 @@ +--- + +ceph_dev: True +cluster: test +public_network: "192.168.1.0/24" +cluster_network: "192.168.2.0/24" +monitor_interface: eth1 +journal_size: 100 +osd_objectstore: "filestore" +ceph_origin: 'repository' +ceph_repository: 'dev' +copy_admin_key: true +os_tuning_params: + - { name: kernel.pid_max, value: 4194303 } + - { name: fs.file-max, value: 26234859 } +ceph_conf_overrides: + global: + osd_pool_default_pg_num: 8 + osd_pool_default_size: 1 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/host_vars/osd0.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/host_vars/osd0.yml new file mode 100644 index 000000000..2e1c7ee9e --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/host_vars/osd0.yml @@ -0,0 +1,7 @@ +--- + +devices: + - '/dev/sdb' +dedicated_devices: + - '/dev/sdc' +osd_scenario: "non-collocated" diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/host_vars/osd1.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/host_vars/osd1.yml new file mode 100644 index 000000000..7e90071c9 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/host_vars/osd1.yml @@ -0,0 +1,6 @@ +--- + +devices: + - '/dev/sdb' + - '/dev/sdc' +osd_scenario: "collocated" diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/hosts b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/hosts new file mode 100644 index 000000000..e0c08b946 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/hosts @@ -0,0 +1,9 @@ +[mons] +mon0 monitor_interface=eth1 + +[osds] +osd0 +osd1 + +[mgrs] +mon0 diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/test.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/test.yml new file mode 100644 index 000000000..24e2c0353 --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/test.yml @@ -0,0 +1,31 @@ +--- + +- hosts: osds + become: yes + tasks: + + - name: list all OSD directories + find: + paths: /var/lib/ceph/osd + file_type: directory + register: osd_paths + + - name: scan all OSD directories + command: "ceph-volume --cluster={{ cluster }} simple scan {{ item.path }}" + environment: + CEPH_VOLUME_DEBUG: 1 + with_items: + - "{{ osd_paths.files }}" + + - name: list all OSD JSON files + find: + paths: /etc/ceph/osd + file_type: file + register: osd_configs + + - name: activate all scanned OSDs + command: "ceph-volume --cluster={{ cluster }} simple activate --file {{ item.path }}" + environment: + CEPH_VOLUME_DEBUG: 1 + with_items: + - "{{ osd_configs.files }}" diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/vagrant_variables.yml b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/vagrant_variables.yml new file mode 100644 index 000000000..b4aa759ab --- /dev/null +++ b/ceph/src/ceph-volume/ceph_volume/tests/functional/simple/xenial/filestore/activate/vagrant_variables.yml @@ -0,0 +1,73 @@ +--- + +# DEPLOY CONTAINERIZED DAEMONS +docker: false + +# DEFINE THE NUMBER OF VMS TO RUN +mon_vms: 1 +osd_vms: 2 +mds_vms: 0 +rgw_vms: 0 +nfs_vms: 0 +rbd_mirror_vms: 0 +client_vms: 0 +iscsi_gw_vms: 0 +mgr_vms: 0 + + +# INSTALL SOURCE OF CEPH +# valid values are 'stable' and 'dev' +ceph_install_source: stable + +# SUBNETS TO USE FOR THE VMS +public_subnet: 192.168.1 +cluster_subnet: 192.168.2 + +# MEMORY +# set 1024 for CentOS +memory: 512 + +# Ethernet interface name +# use eth1 for libvirt and ubuntu precise, enp0s8 for CentOS and ubuntu xenial +eth: 'eth1' + +# Disks +# For libvirt use disks: "[ '/dev/vdb', '/dev/vdc' ]" +# For CentOS7 use disks: "[ '/dev/sda', '/dev/sdb' ]" +disks: "[ '/dev/sdb', '/dev/sdc' ]" + +# VAGRANT BOX +# Ceph boxes are *strongly* suggested. They are under better control and will +# not get updated frequently unless required for build systems. These are (for +# now): +# +# * ceph/ubuntu-xenial +# +# Ubuntu: ceph/ubuntu-xenial bento/ubuntu-16.04 or ubuntu/trusty64 or ubuntu/wily64 +# CentOS: bento/centos-7.1 or puppetlabs/centos-7.0-64-puppet +# libvirt CentOS: centos/7 +# parallels Ubuntu: parallels/ubuntu-14.04 +# Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller' +# For more boxes have a look at: +# - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q= +# - https://download.gluster.org/pub/gluster/purpleidea/vagrant/ +vagrant_box: ceph/ubuntu-xenial +#ssh_private_key_path: "~/.ssh/id_rsa" +# The sync directory changes based on vagrant box +# Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant +#vagrant_sync_dir: /home/vagrant/sync +#vagrant_sync_dir: / +# Disables synced folder creation. Not needed for testing, will skip mounting +# the vagrant directory on the remote box regardless of the provider. +vagrant_disable_synced_folder: true +# VAGRANT URL +# This is a URL to download an image from an alternate location. vagrant_box +# above should be set to the filename of the image. +# Fedora virtualbox: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box +# Fedora libvirt: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-libvirt.box +# vagrant_box_url: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box + +os_tuning_params: + - { name: kernel.pid_max, value: 4194303 } + - { name: fs.file-max, value: 26234859 } + diff --git a/ceph/src/ceph-volume/ceph_volume/tests/functional/xenial/create/Vagrantfile b/ceph/src/ceph-volume/ceph_volume/tests/functional/xenial/create/Vagrantfile deleted file mode 120000 index 2572fa2c9..000000000 --- a/ceph/src/ceph-volume/ceph_volume/tests/functional/xenial/create/Vagrantfile +++ /dev/null @@ -1 +0,0 @@ -../../Vagrantfile \ No newline at end of file diff --git a/ceph/src/ceph-volume/ceph_volume/tests/util/test_arg_validators.py b/ceph/src/ceph-volume/ceph_volume/tests/util/test_arg_validators.py index 917469128..22b962b1d 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/util/test_arg_validators.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/util/test_arg_validators.py @@ -1,10 +1,11 @@ import pytest import argparse +from ceph_volume import exceptions from ceph_volume.util import arg_validators invalid_lv_paths = [ - '', 'lv_name', '///', '/lv_name', 'lv_name/', + '', 'lv_name', '/lv_name', 'lv_name/', '/dev/lv_group/lv_name' ] @@ -22,3 +23,31 @@ class TestLVPath(object): def test_is_valid(self): path = 'vg/lv' assert self.validator(path) == path + + def test_abspath_is_valid(self): + path = '/' + assert self.validator(path) == path + + +class TestOSDPath(object): + + def setup(self): + self.validator = arg_validators.OSDPath() + + def test_is_not_root(self): + with pytest.raises(exceptions.SuperUserError): + self.validator('') + + def test_path_is_not_a_directory(self, is_root, tmpfile, monkeypatch): + monkeypatch.setattr(arg_validators.disk, 'is_partition', lambda x: False) + validator = arg_validators.OSDPath() + with pytest.raises(argparse.ArgumentError): + validator(tmpfile()) + + def test_files_are_missing(self, is_root, tmpdir, monkeypatch): + tmppath = str(tmpdir) + monkeypatch.setattr(arg_validators.disk, 'is_partition', lambda x: False) + validator = arg_validators.OSDPath() + with pytest.raises(argparse.ArgumentError) as error: + validator(tmppath) + assert 'Required file (ceph_fsid) was not found in OSD' in str(error) diff --git a/ceph/src/ceph-volume/ceph_volume/tests/util/test_system.py b/ceph/src/ceph-volume/ceph_volume/tests/util/test_system.py index 7cb6a1f14..56b88b3f4 100644 --- a/ceph/src/ceph-volume/ceph_volume/tests/util/test_system.py +++ b/ceph/src/ceph-volume/ceph_volume/tests/util/test_system.py @@ -1,6 +1,7 @@ import os import pwd import getpass +import pytest from textwrap import dedent from ceph_volume.util import system @@ -34,7 +35,74 @@ class TestMkdirP(object): assert os.path.isdir(path) -class TestIsMounted(object): +@pytest.fixture +def fake_proc(tmpdir, monkeypatch): + PROCDIR = str(tmpdir) + proc_path = os.path.join(PROCDIR, 'mounts') + with open(proc_path, 'w') as f: + f.write(dedent("""nfsd /proc/fs/nfsd nfsd rw,relatime 0 0 + rootfs / rootfs rw 0 0 + sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 + proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 + devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=238292k,nr_inodes=59573,mode=755 0 0 + securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 + tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 + devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 + tmpfs /run tmpfs rw,seclabel,nosuid,nodev,mode=755 0 0 + tmpfs /sys/fs/cgroup tmpfs ro,seclabel,nosuid,nodev,noexec,mode=755 0 0 + cgroup /sys/fs/cgroup/systemd cgroup rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd 0 0 + cgroup /sys/fs/cgroup/freezer cgroup rw,nosuid,nodev,noexec,relatime,freezer 0 0 + configfs /sys/kernel/config configfs rw,relatime 0 0 + /dev/mapper/VolGroup00-LogVol00 / xfs rw,seclabel,relatime,attr2,inode64,noquota 0 0 + selinuxfs /sys/fs/selinux selinuxfs rw,relatime 0 0 + debugfs /sys/kernel/debug debugfs rw,relatime 0 0 + hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime 0 0 + mqueue /dev/mqueue mqueue rw,seclabel,relatime 0 0 + sunrpc /far/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 + /dev/sde4 /two/field/path + nfsd /proc/fs/nfsd nfsd rw,relatime 0 0 + /dev/sde2 /boot xfs rw,seclabel,relatime,attr2,inode64,noquota 0 0 + tmpfs /far/lib/ceph/osd/ceph-5 tmpfs rw,seclabel,relatime 0 0 + tmpfs /far/lib/ceph/osd/ceph-7 tmpfs rw,seclabel,relatime 0 0 + /dev/sda1 /far/lib/ceph/osd/ceph-0 xfs rw,seclabel,noatime,attr2,inode64,noquota 0 0 + tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=50040k,mode=700,uid=1000,gid=1000 0 0 + /dev/sdc2 /boot xfs rw,seclabel,relatime,attr2,inode64,noquota 0 0 + tmpfs /run/user/1000 tmpfs rw,seclabel,mode=700,uid=1000,gid=1000 0 0""")) + monkeypatch.setattr(system, 'PROCDIR', PROCDIR) + monkeypatch.setattr(os.path, 'exists', lambda x: True) + + +class TestPathIsMounted(object): + + def test_is_mounted(self, fake_proc): + assert system.path_is_mounted('/boot') is True + + def test_is_not_mounted(self, fake_proc): + assert system.path_is_mounted('/far/fib/feph') is False + + def test_is_not_mounted_at_destination(self, fake_proc): + assert system.path_is_mounted('/boot', destination='/dev/sda1') is False + + def test_is_mounted_at_destination(self, fake_proc): + assert system.path_is_mounted('/boot', destination='/dev/sdc2') is True + + +class TestDeviceIsMounted(object): + + def test_is_mounted(self, fake_proc): + assert system.device_is_mounted('/dev/sda1') is True + + def test_path_is_not_device(self, fake_proc): + assert system.device_is_mounted('/far/lib/ceph/osd/ceph-7') is False + + def test_is_not_mounted_at_destination(self, fake_proc): + assert system.device_is_mounted('/dev/sda1', destination='/far/lib/ceph/osd/test-1') is False + + def test_is_mounted_at_destination(self, fake_proc): + assert system.device_is_mounted('/dev/sda1', destination='/far/lib/ceph/osd/ceph-7') is False + + +class TestGetMounts(object): def test_not_mounted(self, tmpdir, monkeypatch): PROCDIR = str(tmpdir) @@ -42,48 +110,47 @@ class TestIsMounted(object): with open(proc_path, 'w') as f: f.write('') monkeypatch.setattr(system, 'PROCDIR', PROCDIR) - assert system.is_mounted('sdb') is False + assert system.get_mounts() == {} - def test_is_mounted_(self, tmpdir, monkeypatch): - PROCDIR = str(tmpdir) - proc_path = os.path.join(PROCDIR, 'mounts') - with open(proc_path, 'w') as f: - f.write(dedent("""nfsd /proc/fs/nfsd nfsd rw,relatime 0 0 - /dev/sdc2 /boot xfs rw,seclabel,relatime,attr2,inode64,noquota 0 0 - tmpfs /run/user/1000 tmpfs rw,seclabel,mode=700,uid=1000,gid=1000 0 0""")) - monkeypatch.setattr(system, 'PROCDIR', PROCDIR) - monkeypatch.setattr(os.path, 'exists', lambda x: True) - assert system.is_mounted('/dev/sdc2') is True + def test_is_mounted_(self, fake_proc): + result = system.get_mounts() + assert result['/dev/sdc2'] == ['/boot'] - def test_ignores_two_fields(self, tmpdir, monkeypatch): - PROCDIR = str(tmpdir) - proc_path = os.path.join(PROCDIR, 'mounts') - with open(proc_path, 'w') as f: - f.write(dedent("""nfsd /proc/fs/nfsd nfsd rw,relatime 0 0 - /dev/sdc2 /boot - tmpfs /run/user/1000 tmpfs rw,seclabel,mode=700,uid=1000,gid=1000 0 0""")) - monkeypatch.setattr(system, 'PROCDIR', PROCDIR) - monkeypatch.setattr(os.path, 'exists', lambda x: True) - assert system.is_mounted('/dev/sdc2') is False + def test_ignores_two_fields(self, fake_proc): + result = system.get_mounts() + assert result.get('/dev/sde4') is None - def test_not_mounted_at_destination(self, tmpdir, monkeypatch): - PROCDIR = str(tmpdir) - proc_path = os.path.join(PROCDIR, 'mounts') - with open(proc_path, 'w') as f: - f.write(dedent("""nfsd /proc/fs/nfsd nfsd rw,relatime 0 0 - /dev/sdc2 /var/lib/ceph/osd/ceph-9 xfs rw,attr2,inode64,noquota 0 0 - tmpfs /run/user/1000 tmpfs rw,seclabel,mode=700,uid=1000,gid=1000 0 0""")) - monkeypatch.setattr(system, 'PROCDIR', PROCDIR) - monkeypatch.setattr(os.path, 'exists', lambda x: True) - assert system.is_mounted('/dev/sdc2', '/var/lib/ceph/osd/ceph-0') is False + def test_tmpfs_is_reported(self, fake_proc): + result = system.get_mounts() + assert result['tmpfs'][0] == '/dev/shm' + + def test_non_skip_devs_arent_reported(self, fake_proc): + result = system.get_mounts() + assert result.get('cgroup') is None + + def test_multiple_mounts_are_appended(self, fake_proc): + result = system.get_mounts() + assert len(result['tmpfs']) == 7 - def test_is_mounted_at_destination(self, tmpdir, monkeypatch): + def test_nonexistent_devices_are_skipped(self, tmpdir, monkeypatch): PROCDIR = str(tmpdir) proc_path = os.path.join(PROCDIR, 'mounts') with open(proc_path, 'w') as f: f.write(dedent("""nfsd /proc/fs/nfsd nfsd rw,relatime 0 0 - /dev/sdc2 /var/lib/ceph/osd/ceph-0 xfs rw,attr2,inode64,noquota 0 0 - tmpfs /run/user/1000 tmpfs rw,seclabel,mode=700,uid=1000,gid=1000 0 0""")) + /dev/sda1 /far/lib/ceph/osd/ceph-0 xfs rw,attr2,inode64,noquota 0 0 + /dev/sda2 /far/lib/ceph/osd/ceph-1 xfs rw,attr2,inode64,noquota 0 0""")) monkeypatch.setattr(system, 'PROCDIR', PROCDIR) - monkeypatch.setattr(os.path, 'exists', lambda x: True) - assert system.is_mounted('/dev/sdc2', '/var/lib/ceph/osd/ceph-0') is True + monkeypatch.setattr(os.path, 'exists', lambda x: False if x == '/dev/sda1' else True) + result = system.get_mounts() + assert result.get('/dev/sda1') is None + + +class TestIsBinary(object): + + def test_is_binary(self, tmpfile): + binary_path = tmpfile(contents='asd\n\nlkjh\x00') + assert system.is_binary(binary_path) + + def test_is_not_binary(self, tmpfile): + binary_path = tmpfile(contents='asd\n\nlkjh0') + assert system.is_binary(binary_path) is False diff --git a/ceph/src/ceph-volume/ceph_volume/util/arg_validators.py b/ceph/src/ceph-volume/ceph_volume/util/arg_validators.py index feb470716..349d5da17 100644 --- a/ceph/src/ceph-volume/ceph_volume/util/arg_validators.py +++ b/ceph/src/ceph-volume/ceph_volume/util/arg_validators.py @@ -1,4 +1,8 @@ import argparse +import os +from ceph_volume import terminal +from ceph_volume import decorators +from ceph_volume.util import disk class LVPath(object): @@ -7,12 +11,20 @@ class LVPath(object): / + Or a full path to a device, like ``/dev/sda`` + Because for LVM it is better to be specific on what group does an lv belongs to. """ def __call__(self, string): error = None + if string.startswith('/'): + if not os.path.exists(string): + error = "Argument (device) does not exist: %s" % string + raise argparse.ArgumentError(None, error) + else: + return string try: vg, lv = string.split('/') except ValueError: @@ -27,3 +39,35 @@ class LVPath(object): if error: raise argparse.ArgumentError(None, error) return string + + +class OSDPath(object): + """ + Validate path exists and it looks like an OSD directory. + """ + + @decorators.needs_root + def __call__(self, string): + if not os.path.exists(string): + error = "Path does not exist: %s" % string + raise argparse.ArgumentError(None, error) + + arg_is_partition = disk.is_partition(string) + if arg_is_partition: + return os.path.abspath(string) + absolute_path = os.path.abspath(string) + if not os.path.isdir(absolute_path): + error = "Argument is not a directory or device which is required to scan" + raise argparse.ArgumentError(None, error) + key_files = ['ceph_fsid', 'fsid', 'keyring', 'ready', 'type', 'whoami'] + dir_files = os.listdir(absolute_path) + for key_file in key_files: + if key_file not in dir_files: + terminal.error('All following files must exist in path: %s' % ' '.join(key_files)) + error = "Required file (%s) was not found in OSD dir path: %s" % ( + key_file, + absolute_path + ) + raise argparse.ArgumentError(None, error) + + return os.path.abspath(string) diff --git a/ceph/src/ceph-volume/ceph_volume/util/disk.py b/ceph/src/ceph-volume/ceph_volume/util/disk.py index 0d3061d3c..da3dc9341 100644 --- a/ceph/src/ceph-volume/ceph_volume/util/disk.py +++ b/ceph/src/ceph-volume/ceph_volume/util/disk.py @@ -1,3 +1,5 @@ +import os +import stat from ceph_volume import process @@ -22,3 +24,160 @@ def get_device_from_partuuid(partuuid): ['sudo', 'blkid', '-t', 'PARTUUID="%s"' % partuuid, '-o', 'device'] ) return ' '.join(out).strip() + + +def _stat_is_device(stat_obj): + """ + Helper function that will interpret ``os.stat`` output directly, so that other + functions can call ``os.stat`` once and interpret that result several times + """ + return stat.S_ISBLK(stat_obj) + + +def lsblk(device, columns=None): + """ + Create a dictionary of identifying values for a device using ``lsblk``. + Each supported column is a key, in its *raw* format (all uppercase + usually). ``lsblk`` has support for certain "columns" (in blkid these + would be labels), and these columns vary between distributions and + ``lsblk`` versions. The newer versions support a richer set of columns, + while older ones were a bit limited. + + These are the default lsblk columns reported which are safe to use for + Ubuntu 14.04.5 LTS: + + NAME device name + KNAME internal kernel device name + MAJ:MIN major:minor device number + FSTYPE filesystem type + MOUNTPOINT where the device is mounted + LABEL filesystem LABEL + UUID filesystem UUID + RO read-only device + RM removable device + MODEL device identifier + SIZE size of the device + STATE state of the device + OWNER user name + GROUP group name + MODE device node permissions + ALIGNMENT alignment offset + MIN-IO minimum I/O size + OPT-IO optimal I/O size + PHY-SEC physical sector size + LOG-SEC logical sector size + ROTA rotational device + SCHED I/O scheduler name + RQ-SIZE request queue size + TYPE device type + DISC-ALN discard alignment offset + DISC-GRAN discard granularity + DISC-MAX discard max bytes + DISC-ZERO discard zeroes data + + There is a bug in ``lsblk`` where using all the available (supported) + columns will result in no output (!), in order to workaround this the + following columns have been removed from the default reporting columns: + + * RQ-SIZE (request queue size) + * MIN-IO minimum I/O size + * OPT-IO optimal I/O size + + These should be available however when using `columns`. For example:: + + >>> lsblk('/dev/sda1', columns=['OPT-IO']) + {'OPT-IO': '0'} + + Normal CLI output, as filtered by the flags in this function will look like :: + + $ sudo lsblk --nodeps -P -o NAME,KNAME,MAJ:MIN,FSTYPE,MOUNTPOINT + NAME="sda1" KNAME="sda1" MAJ:MIN="8:1" FSTYPE="ext4" MOUNTPOINT="/" + + :param columns: A list of columns to report as keys in its original form. + """ + default_columns = [ + 'NAME', 'KNAME', 'MAJ:MIN', 'FSTYPE', 'MOUNTPOINT', 'LABEL', 'UUID', + 'RO', 'RM', 'MODEL', 'SIZE', 'STATE', 'OWNER', 'GROUP', 'MODE', + 'ALIGNMENT', 'PHY-SEC', 'LOG-SEC', 'ROTA', 'SCHED', 'TYPE', 'DISC-ALN', + 'DISC-GRAN', 'DISC-MAX', 'DISC-ZERO' + ] + device = device.rstrip('/') + columns = columns or default_columns + # --nodeps -> Avoid adding children/parents to the device, only give information + # on the actual device we are querying for + # -P -> Produce pairs of COLUMN="value" + # -o -> Use the columns specified or default ones provided by this function + command = ['sudo', 'lsblk', '--nodeps', '-P', '-o'] + command.append(','.join(columns)) + command.append(device) + out, err, rc = process.call(command) + + if rc != 0: + return {} + + # parse the COLUMN="value" output to construct the dictionary + pairs = ' '.join(out).split() + parsed = {} + for pair in pairs: + try: + column, value = pair.split('=') + except ValueError: + continue + parsed[column] = value.strip().strip().strip('"') + return parsed + + +def _lsblk_type(device): + """ + Helper function that will use the ``TYPE`` label output of ``lsblk`` to determine + if a device is a partition or disk. + It does not process the output to return a boolean, but it does process it to return the + """ + out, err, rc = process.call( + ['sudo', 'blkid', '-s', 'PARTUUID', '-o', 'value', device] + ) + return ' '.join(out).strip() + + +def is_device(dev): + """ + Boolean to determine if a given device is a block device (**not** + a partition!) + + For example: /dev/sda would return True, but not /dev/sdc1 + """ + if not os.path.exists(dev): + return False + # use lsblk first, fall back to using stat + TYPE = lsblk(dev).get('TYPE') + if TYPE: + return TYPE == 'disk' + + # fallback to stat + return _stat_is_device(os.lstat(dev).st_mode) + if stat.S_ISBLK(os.lstat(dev)): + return True + return False + + +def is_partition(dev): + """ + Boolean to determine if a given device is a partition, like /dev/sda1 + """ + if not os.path.exists(dev): + return False + # use lsblk first, fall back to using stat + TYPE = lsblk(dev).get('TYPE') + if TYPE: + return TYPE == 'part' + + # fallback to stat + stat_obj = os.stat(dev) + if _stat_is_device(stat_obj.st_mode): + return False + + major = os.major(stat_obj.st_rdev) + minor = os.minor(stat_obj.st_rdev) + if os.path.exists('/sys/dev/block/%d:%d/partition' % (major, minor)): + return True + return False diff --git a/ceph/src/ceph-volume/ceph_volume/util/prepare.py b/ceph/src/ceph-volume/ceph_volume/util/prepare.py index eefa0adc2..6b38fe097 100644 --- a/ceph/src/ceph-volume/ceph_volume/util/prepare.py +++ b/ceph/src/ceph-volume/ceph_volume/util/prepare.py @@ -57,8 +57,21 @@ def create_id(fsid, json_secrets): return ' '.join(stdout).strip() -def create_path(osd_id): +def mount_tmpfs(path): + process.run([ + 'sudo', + 'mount', + '-t', + 'tmpfs', 'tmpfs', + path + ]) + + +def create_osd_path(osd_id, tmpfs=False): + path = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id) system.mkdir_p('/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id)) + if tmpfs: + mount_tmpfs(path) def format_device(device): @@ -98,15 +111,39 @@ def mount_osd(device, osd_id): process.run(command) -def link_journal(journal_device, osd_id): - journal_path = '/var/lib/ceph/osd/%s-%s/journal' % ( +def _link_device(device, device_type, osd_id): + """ + Allow linking any device type in an OSD directory. ``device`` must the be + source, with an absolute path and ``device_type`` will be the destination + name, like 'journal', or 'block' + """ + device_path = '/var/lib/ceph/osd/%s-%s/%s' % ( conf.cluster, - osd_id + osd_id, + device_type ) - command = ['sudo', 'ln', '-s', journal_device, journal_path] + command = ['sudo', 'ln', '-s', device, device_path] + system.chown(device) + process.run(command) +def link_journal(journal_device, osd_id): + _link_device(journal_device, 'journal', osd_id) + + +def link_block(block_device, osd_id): + _link_device(block_device, 'block', osd_id) + + +def link_wal(wal_device, osd_id): + _link_device(wal_device, 'block.wal', osd_id) + + +def link_db(db_device, osd_id): + _link_device(db_device, 'block.db', osd_id) + + def get_monmap(osd_id): """ Before creating the OSD files, a monmap needs to be retrieved so that it @@ -130,7 +167,64 @@ def get_monmap(osd_id): ]) -def osd_mkfs(osd_id, fsid): +def osd_mkfs_bluestore(osd_id, fsid, keyring=None, wal=False, db=False): + """ + Create the files for the OSD to function. A normal call will look like: + + ceph-osd --cluster ceph --mkfs --mkkey -i 0 \ + --monmap /var/lib/ceph/osd/ceph-0/activate.monmap \ + --osd-data /var/lib/ceph/osd/ceph-0 \ + --osd-uuid 8d208665-89ae-4733-8888-5d3bfbeeec6c \ + --keyring /var/lib/ceph/osd/ceph-0/keyring \ + --setuser ceph --setgroup ceph + + In some cases it is required to use the keyring, when it is passed in as + a keywork argument it is used as part of the ceph-osd command + """ + path = '/var/lib/ceph/osd/%s-%s/' % (conf.cluster, osd_id) + monmap = os.path.join(path, 'activate.monmap') + + system.chown(path) + + base_command = [ + 'sudo', + 'ceph-osd', + '--cluster', conf.cluster, + # undocumented flag, sets the `type` file to contain 'bluestore' + '--osd-objectstore', 'bluestore', + '--mkfs', + '-i', osd_id, + '--monmap', monmap, + ] + + supplementary_command = [ + '--osd-data', path, + '--osd-uuid', fsid, + '--setuser', 'ceph', + '--setgroup', 'ceph' + ] + + if keyring is not None: + base_command.extend(['--key', keyring]) + + if wal: + base_command.extend( + ['--bluestore-block-wal-path', wal] + ) + system.chown(wal) + + if db: + base_command.extend( + ['--bluestore-block-db-path', db] + ) + system.chown(db) + + command = base_command + supplementary_command + + process.run(command, obfuscate='--key') + + +def osd_mkfs_filestore(osd_id, fsid): """ Create the files for the OSD to function. A normal call will look like: @@ -154,6 +248,8 @@ def osd_mkfs(osd_id, fsid): 'sudo', 'ceph-osd', '--cluster', conf.cluster, + # undocumented flag, sets the `type` file to contain 'filestore' + '--osd-objectstore', 'filestore', '--mkfs', '-i', osd_id, '--monmap', monmap, diff --git a/ceph/src/ceph-volume/ceph_volume/util/system.py b/ceph/src/ceph-volume/ceph_volume/util/system.py index 084a0e0d3..d580a4c28 100644 --- a/ceph/src/ceph-volume/ceph_volume/util/system.py +++ b/ceph/src/ceph-volume/ceph_volume/util/system.py @@ -2,6 +2,7 @@ import errno import os import pwd import platform +import tempfile import uuid from ceph_volume import process from . import as_string @@ -68,37 +69,122 @@ def chown(path, recursive=True): os.chown(path, uid, gid) -def is_mounted(source, destination=None): +def is_binary(path): """ - Check if the given device is mounted, optionally validating destination. - This relies on absolute path devices, it will ignore non-absolute - entries like:: + Detect if a file path is a binary or not. Will falsely report as binary + when utf-16 encoded. In the ceph universe there is no such risk (yet) + """ + with open(path, 'rb') as fp: + contents = fp.read(8192) + if b'\x00' in contents: # a null byte may signal binary + return True + return False + + +class tmp_mount(object): + """ + Temporarily mount a device on a temporary directory, + and unmount it upon exit + """ + + def __init__(self, device): + self.device = device + self.path = None + + def __enter__(self): + self.path = tempfile.mkdtemp() + process.run([ + 'sudo', + 'mount', + '-v', + self.device, + self.path + ]) + return self.path + + def __exit__(self, exc_type, exc_val, exc_tb): + process.run([ + 'sudo', + 'umount', + '-v', + self.path + ]) + + +def path_is_mounted(path, destination=None): + """ + Check if the given path is mounted + """ + mounts = get_mounts(paths=True) + realpath = os.path.realpath(path) + mounted_locations = mounts.get(realpath, []) + + if destination: + if destination.startswith('/'): + destination = os.path.realpath(destination) + return destination in mounted_locations + return mounted_locations != [] + - tmpfs /run tmpfs rw,seclabel,nosuid,nodev,mode=755 0 0 +def device_is_mounted(dev, destination=None): + """ + Check if the given device is mounted, optionally validating that a + destination exists + """ + mounts = get_mounts(devices=True) + realpath = os.path.realpath(dev) if dev.startswith('/') else dev + destination = os.path.realpath(destination) if destination else None + mounted_locations = mounts.get(realpath, []) + + if destination: + return destination in mounted_locations + return mounted_locations != [] + + +def get_mounts(devices=False, paths=False): + """ + Create a mapping of all available system mounts so that other helpers can + detect nicely what path or device is mounted - But will parse paths that are absolute like:: + It ignores (most of) non existing devices, but since some setups might need + some extra device information, it will make an exception for: - /dev/sdc2 /boot xfs rw,attr2,inode64,noquota 0 0 + - tmpfs + - devtmpfs - When destination is passed in, it will check that the entry where the - source appears is mounted to where destination defines. This is useful so - that an error message can report that a source is not mounted at an - expected destination. + If ``devices`` is set to ``True`` the mapping will be a device-to-path(s), + if ``paths`` is set to ``True`` then the mapping will be + a path-to-device(s) """ - dev = os.path.realpath(source) - with open(PROCDIR + '/mounts', 'rb') as proc_mounts: - for line in proc_mounts: - fields = line.split() - if len(fields) < 3: + devices_mounted = {} + paths_mounted = {} + do_not_skip = ['tmpfs', 'devtmpfs'] + default_to_devices = devices is False and paths is False + + with open(PROCDIR + '/mounts', 'rb') as mounts: + proc_mounts = mounts.readlines() + + for line in proc_mounts: + fields = [as_string(f) for f in line.split()] + if len(fields) < 3: + continue + device = os.path.realpath(fields[0]) if fields[0].startswith('/') else fields[0] + path = os.path.realpath(fields[1]) + # only care about actual existing devices + if not os.path.exists(device) or not device.startswith('/'): + if device not in do_not_skip: continue - mounted_device = fields[0] - mounted_path = fields[1] - if os.path.isabs(mounted_device) and os.path.exists(mounted_device): - mounted_device = os.path.realpath(mounted_device) - if as_string(mounted_device) == dev: - if destination: - destination = os.path.realpath(destination) - return destination == as_string(os.path.realpath(mounted_path)) - else: - return True - return False + if device in devices_mounted.keys(): + devices_mounted[device].append(path) + else: + devices_mounted[device] = [path] + if path in paths_mounted.keys(): + paths_mounted[path].append(device) + else: + paths_mounted[path] = [device] + + # Default to returning information for devices if + if devices is True or default_to_devices: + return devices_mounted + else: + return paths_mounted diff --git a/ceph/src/ceph.in b/ceph/src/ceph.in index cb1bd8694..7c1eda2c0 100755 --- a/ceph/src/ceph.in +++ b/ceph/src/ceph.in @@ -48,7 +48,7 @@ PRIO_USEFUL = 5 PRIO_UNINTERESTING = 2 PRIO_DEBUGONLY = 0 -PRIO_DEFAULT = PRIO_USEFUL +PRIO_DEFAULT = PRIO_INTERESTING # Make life easier on developers: # If our parent dir contains CMakeCache.txt and bin/init-ceph, @@ -228,7 +228,7 @@ def validate_target(target): file=sys.stderr) return False - if service_id in exist_ids: + if service_id in exist_ids or len(exist_ids) > 0 and service_id == '*': return True else: print('WARN: the service id you provided does not exist. service id should ' diff --git a/ceph/src/ceph_mgr.cc b/ceph/src/ceph_mgr.cc index 91043f6e8..5e8f6798e 100644 --- a/ceph/src/ceph_mgr.cc +++ b/ceph/src/ceph_mgr.cc @@ -16,7 +16,10 @@ #include +#include + #include "include/types.h" +#include "include/compat.h" #include "common/config.h" #include "common/ceph_argparse.h" #include "common/errno.h" @@ -38,6 +41,8 @@ static void usage() */ int main(int argc, const char **argv) { + ceph_pthread_setname(pthread_self(), "ceph-mgr"); + vector args; argv_to_vec(argc, argv, args); env_to_vec(args); diff --git a/ceph/src/ceph_mon.cc b/ceph/src/ceph_mon.cc index 3663bb04e..41a6ee0eb 100644 --- a/ceph/src/ceph_mon.cc +++ b/ceph/src/ceph_mon.cc @@ -247,7 +247,6 @@ int main(int argc, const char **argv) flags, "mon_data"); ceph_heap_profiler_init(); - uuid_d fsid; std::string val; for (std::vector::iterator i = args.begin(); i != args.end(); ) { if (ceph_argparse_double_dash(args, i)) { @@ -331,10 +330,11 @@ int main(int argc, const char **argv) MonMap monmap; // load or generate monmap - if (g_conf->monmap.length()) { - int err = monmapbl.read_file(g_conf->monmap.c_str(), &error); + const auto monmap_fn = g_conf->get_val("monmap"); + if (monmap_fn.length()) { + int err = monmapbl.read_file(monmap_fn.c_str(), &error); if (err < 0) { - derr << argv[0] << ": error reading " << g_conf->monmap << ": " << error << dendl; + derr << argv[0] << ": error reading " << monmap_fn << ": " << error << dendl; exit(1); } try { @@ -342,9 +342,8 @@ int main(int argc, const char **argv) // always mark seed/mkfs monmap as epoch 0 monmap.set_epoch(0); - } - catch (const buffer::error& e) { - derr << argv[0] << ": error decoding monmap " << g_conf->monmap << ": " << e.what() << dendl; + } catch (const buffer::error& e) { + derr << argv[0] << ": error decoding monmap " << monmap_fn << ": " << e.what() << dendl; exit(1); } } else { @@ -393,9 +392,10 @@ int main(int argc, const char **argv) } } - if (!g_conf->fsid.is_zero()) { - monmap.fsid = g_conf->fsid; - dout(0) << argv[0] << ": set fsid to " << g_conf->fsid << dendl; + const auto fsid = g_conf->get_val("fsid"); + if (!fsid.is_zero()) { + monmap.fsid = fsid; + dout(0) << argv[0] << ": set fsid to " << fsid << dendl; } if (monmap.fsid.is_zero()) { diff --git a/ceph/src/ceph_osd.cc b/ceph/src/ceph_osd.cc index d7e54a3a3..1cfda9c1d 100644 --- a/ceph/src/ceph_osd.cc +++ b/ceph/src/ceph_osd.cc @@ -266,29 +266,6 @@ int main(int argc, const char **argv) cephd_preload_embedded_plugins(); #endif - if (mkfs) { - common_init_finish(g_ceph_context); - MonClient mc(g_ceph_context); - if (mc.build_initial_monmap() < 0) - return -1; - if (mc.get_monmap_privately() < 0) - return -1; - - if (mc.monmap.fsid.is_zero()) { - derr << "must specify cluster fsid" << dendl; - return -EINVAL; - } - - int err = OSD::mkfs(g_ceph_context, store, g_conf->osd_data, - mc.monmap.fsid, whoami); - if (err < 0) { - derr << TEXT_RED << " ** ERROR: error creating empty object store in " - << g_conf->osd_data << ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl; - exit(1); - } - derr << "created object store " << g_conf->osd_data - << " for osd." << whoami << " fsid " << mc.monmap.fsid << dendl; - } if (mkkey) { common_init_finish(g_ceph_context); KeyRing *keyring = KeyRing::create_empty(); @@ -317,6 +294,29 @@ int main(int argc, const char **argv) derr << "created new key in keyring " << g_conf->keyring << dendl; } } + if (mkfs) { + common_init_finish(g_ceph_context); + MonClient mc(g_ceph_context); + if (mc.build_initial_monmap() < 0) + return -1; + if (mc.get_monmap_privately() < 0) + return -1; + + if (mc.monmap.fsid.is_zero()) { + derr << "must specify cluster fsid" << dendl; + return -EINVAL; + } + + int err = OSD::mkfs(g_ceph_context, store, g_conf->osd_data, + mc.monmap.fsid, whoami); + if (err < 0) { + derr << TEXT_RED << " ** ERROR: error creating empty object store in " + << g_conf->osd_data << ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl; + exit(1); + } + derr << "created object store " << g_conf->osd_data + << " for osd." << whoami << " fsid " << mc.monmap.fsid << dendl; + } if (mkfs || mkkey) exit(0); if (mkjournal) { diff --git a/ceph/src/client/Client.cc b/ceph/src/client/Client.cc index 29a9c49f7..1d9277a61 100644 --- a/ceph/src/client/Client.cc +++ b/ceph/src/client/Client.cc @@ -5944,19 +5944,6 @@ void Client::unmount() ldout(cct, 2) << "unmounted." << dendl; } - - -class C_C_Tick : public Context { - Client *client; -public: - explicit C_C_Tick(Client *c) : client(c) {} - void finish(int r) override { - // Called back via Timer, which takes client_lock for us - assert(client->client_lock.is_locked_by_me()); - client->tick(); - } -}; - void Client::flush_cap_releases() { // send any cap releases @@ -5985,9 +5972,13 @@ void Client::tick() } ldout(cct, 21) << "tick" << dendl; - tick_event = new C_C_Tick(this); - timer.add_event_after(cct->_conf->client_tick_interval, tick_event); - + tick_event = timer.add_event_after( + cct->_conf->client_tick_interval, + new FunctionContext([this](int) { + // Called back via Timer, which takes client_lock for us + assert(client_lock.is_locked_by_me()); + tick(); + })); utime_t now = ceph_clock_now(); if (!mounted && !mds_requests.empty()) { diff --git a/ceph/src/client/Client.h b/ceph/src/client/Client.h index e89a25440..16aef0312 100644 --- a/ceph/src/client/Client.h +++ b/ceph/src/client/Client.h @@ -498,7 +498,6 @@ protected: friend class C_Client_CacheInvalidate; // calls ino_invalidate_cb friend class C_Client_DentryInvalidate; // calls dentry_invalidate_cb friend class C_Block_Sync; // Calls block map and protected helpers - friend class C_C_Tick; // Asserts on client_lock friend class C_Client_RequestInterrupt; friend class C_Client_Remount; friend void intrusive_ptr_release(Inode *in); diff --git a/ceph/src/cls/journal/cls_journal.cc b/ceph/src/cls/journal/cls_journal.cc index f966c07f2..2f1d18d0d 100644 --- a/ceph/src/cls/journal/cls_journal.cc +++ b/ceph/src/cls/journal/cls_journal.cc @@ -188,6 +188,7 @@ int expire_tags(cls_method_context_t hctx, const std::string *skip_client_id) { if (tag.tid >= minimum_tag_tid) { // no need to check for tag classes beyond this point vals.clear(); + more = false; break; } } @@ -1047,6 +1048,7 @@ int journal_tag_list(cls_method_context_t hctx, bufferlist *in, // completed calculation of tag class minimums if (tag.tid >= minimum_tag_tid) { vals.clear(); + more = false; break; } } else if (tag_pass == TAG_PASS_LIST) { diff --git a/ceph/src/cls/rbd/cls_rbd.cc b/ceph/src/cls/rbd/cls_rbd.cc index 79795dbc3..90a48a821 100644 --- a/ceph/src/cls/rbd/cls_rbd.cc +++ b/ceph/src/cls/rbd/cls_rbd.cc @@ -2460,7 +2460,7 @@ int object_map_update(cls_method_context_t hctx, bufferlist *in, bufferlist *out CLS_ERR("object map footer read failed"); return r; } - + try { bufferlist::iterator it = footer_bl.begin(); object_map.decode_footer(it); @@ -2496,13 +2496,14 @@ int object_map_update(cls_method_context_t hctx, bufferlist *in, bufferlist *out } bool updated = false; - for (uint64_t object_no = start_object_no; object_no < end_object_no; - ++object_no) { - uint8_t state = object_map[object_no]; + auto it = object_map.begin() + start_object_no; + auto end_it = object_map.begin() + end_object_no; + for (; it != end_it; ++it) { + uint8_t state = *it; if ((!current_object_state || state == *current_object_state || (*current_object_state == OBJECT_EXISTS && state == OBJECT_EXISTS_CLEAN)) && state != new_object_state) { - object_map[object_no] = new_object_state; + *it = new_object_state; updated = true; } } @@ -3167,6 +3168,22 @@ int uuid_get(cls_method_context_t hctx, std::string *mirror_uuid) { return 0; } +int list_watchers(cls_method_context_t hctx, + std::set *entities) { + obj_list_watch_response_t watchers; + int r = cls_cxx_list_watchers(hctx, &watchers); + if (r < 0 && r != -ENOENT) { + CLS_ERR("error listing watchers: '%s'", cpp_strerror(r).c_str()); + return r; + } + + entities->clear(); + for (auto &w : watchers.entries) { + entities->emplace(w.name, w.addr); + } + return 0; +} + int read_peers(cls_method_context_t hctx, std::vector *peers) { std::string last_read = PEER_KEY_PREFIX; @@ -3419,6 +3436,7 @@ int image_status_remove(cls_method_context_t hctx, } int image_status_get(cls_method_context_t hctx, const string &global_image_id, + const std::set &watchers, cls::rbd::MirrorImageStatus *status) { bufferlist bl; @@ -3441,23 +3459,9 @@ int image_status_get(cls_method_context_t hctx, const string &global_image_id, return -EIO; } - obj_list_watch_response_t watchers; - r = cls_cxx_list_watchers(hctx, &watchers); - if (r < 0 && r != -ENOENT) { - CLS_ERR("error listing watchers: '%s'", cpp_strerror(r).c_str()); - return r; - } *status = static_cast(ondisk_status); - status->up = false; - for (auto &w : watchers.entries) { - if (w.name == ondisk_status.origin.name && - w.addr == ondisk_status.origin.addr) { - status->up = true; - break; - } - } - + status->up = (watchers.find(ondisk_status.origin) != watchers.end()); return 0; } @@ -3469,11 +3473,17 @@ int image_status_list(cls_method_context_t hctx, int max_read = RBD_MAX_KEYS_READ; bool more = true; + std::set watchers; + int r = list_watchers(hctx, &watchers); + if (r < 0) { + return r; + } + while (more && mirror_images->size() < max_return) { std::map vals; CLS_LOG(20, "last_read = '%s'", last_read.c_str()); - int r = cls_cxx_map_get_vals(hctx, last_read, IMAGE_KEY_PREFIX, max_read, - &vals, &more); + r = cls_cxx_map_get_vals(hctx, last_read, IMAGE_KEY_PREFIX, max_read, &vals, + &more); if (r < 0) { CLS_ERR("error reading mirror image directory by name: %s", cpp_strerror(r).c_str()); @@ -3496,7 +3506,8 @@ int image_status_list(cls_method_context_t hctx, (*mirror_images)[image_id] = mirror_image; cls::rbd::MirrorImageStatus status; - int r1 = image_status_get(hctx, mirror_image.global_image_id, &status); + int r1 = image_status_get(hctx, mirror_image.global_image_id, watchers, + &status); if (r1 < 0) { continue; } @@ -3513,20 +3524,12 @@ int image_status_list(cls_method_context_t hctx, int image_status_get_summary(cls_method_context_t hctx, std::map *states) { - obj_list_watch_response_t watchers_; - int r = cls_cxx_list_watchers(hctx, &watchers_); + std::set watchers; + int r = list_watchers(hctx, &watchers); if (r < 0) { - if (r != -ENOENT) { - CLS_ERR("error listing watchers: '%s'", cpp_strerror(r).c_str()); - } return r; } - set watchers; - for (auto &w : watchers_.entries) { - watchers.insert(entity_inst_t(w.name, w.addr)); - } - states->clear(); string last_read = IMAGE_KEY_PREFIX; @@ -3559,7 +3562,7 @@ int image_status_get_summary(cls_method_context_t hctx, } cls::rbd::MirrorImageStatus status; - image_status_get(hctx, mirror_image.global_image_id, &status); + image_status_get(hctx, mirror_image.global_image_id, watchers, &status); cls::rbd::MirrorImageStatusState state = status.up ? status.state : cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN; @@ -3575,20 +3578,12 @@ int image_status_get_summary(cls_method_context_t hctx, } int image_status_remove_down(cls_method_context_t hctx) { - obj_list_watch_response_t watchers_; - int r = cls_cxx_list_watchers(hctx, &watchers_); + std::set watchers; + int r = list_watchers(hctx, &watchers); if (r < 0) { - if (r != -ENOENT) { - CLS_ERR("error listing watchers: '%s'", cpp_strerror(r).c_str()); - } return r; } - set watchers; - for (auto &w : watchers_.entries) { - watchers.insert(entity_inst_t(w.name, w.addr)); - } - string last_read = STATUS_GLOBAL_KEY_PREFIX; int max_read = RBD_MAX_KEYS_READ; bool more = true; @@ -4275,8 +4270,14 @@ int mirror_image_status_get(cls_method_context_t hctx, bufferlist *in, return -EINVAL; } + std::set watchers; + int r = mirror::list_watchers(hctx, &watchers); + if (r < 0) { + return r; + } + cls::rbd::MirrorImageStatus status; - int r = mirror::image_status_get(hctx, global_image_id, &status); + r = mirror::image_status_get(hctx, global_image_id, watchers, &status); if (r < 0) { return r; } diff --git a/ceph/src/cls/rgw/cls_rgw.cc b/ceph/src/cls/rgw/cls_rgw.cc index 17a618053..354a132da 100644 --- a/ceph/src/cls/rgw/cls_rgw.cc +++ b/ceph/src/cls/rgw/cls_rgw.cc @@ -445,8 +445,9 @@ int rgw_bucket_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out) CLS_LOG(20, "entry %s[%s] is not valid\n", key.name.c_str(), key.instance.c_str()); continue; } - - if (!op.list_versions && !entry.is_visible()) { + + // filter out noncurrent versions, delete markers, and initial marker + if (!op.list_versions && (!entry.is_visible() || op.start_obj.name == key.name)) { CLS_LOG(20, "entry %s[%s] is not visible\n", key.name.c_str(), key.instance.c_str()); continue; } @@ -935,6 +936,7 @@ int rgw_bucket_complete_op(cls_method_context_t hctx, bufferlist *in, bufferlist unaccount_entry(header, remove_entry); if (op.log_op && !header.syncstopped) { + ++header.ver; // increment index version, or we'll overwrite keys previously written rc = log_index_operation(hctx, remove_key, CLS_RGW_OP_DEL, op.tag, remove_entry.meta.mtime, remove_entry.ver, CLS_RGW_STATE_COMPLETE, header.ver, header.max_marker, op.bilog_flags, NULL, NULL, &op.zones_trace); if (rc < 0) @@ -1863,7 +1865,8 @@ static int rgw_bucket_clear_olh(cls_method_context_t hctx, bufferlist *in, buffe return 0; } -int rgw_dir_suggest_changes(cls_method_context_t hctx, bufferlist *in, bufferlist *out) +int rgw_dir_suggest_changes(cls_method_context_t hctx, + bufferlist *in, bufferlist *out) { CLS_LOG(1, "rgw_dir_suggest_changes()"); @@ -1956,8 +1959,21 @@ int rgw_dir_suggest_changes(cls_method_context_t hctx, bufferlist *in, bufferlis } break; case CEPH_RGW_UPDATE: + if (!cur_disk.exists) { + // this update would only have been sent by the rgw client + // if the rgw_bucket_dir_entry existed, however between that + // check and now the entry has diappeared, so we were likely + // in the midst of a delete op, and we will not recreate the + // entry + CLS_LOG(10, + "CEPH_RGW_UPDATE not applied because rgw_bucket_dir_entry" + " no longer exists\n"); + break; + } + CLS_LOG(10, "CEPH_RGW_UPDATE name=%s instance=%s total_entries: %" PRId64 " -> %" PRId64 "\n", cur_change.key.name.c_str(), cur_change.key.instance.c_str(), stats.num_entries, stats.num_entries + 1); + stats.num_entries++; stats.total_size += cur_change.meta.accounted_size; stats.total_size_rounded += cls_rgw_get_rounded_size(cur_change.meta.accounted_size); @@ -1978,10 +1994,9 @@ int rgw_dir_suggest_changes(cls_method_context_t hctx, bufferlist *in, bufferlis } } break; - } - } - - } + } // switch(op) + } // if (cur_disk.pending_map.empty()) + } // while (!in_iter.end()) if (header_changed) { return write_bucket_header(hctx, &header); @@ -2900,9 +2915,7 @@ static int usage_iterate_range(cls_method_context_t hctx, uint64_t start, uint64 bool by_user = !user.empty(); uint32_t i = 0; string user_key; - - if (truncated) - *truncated = false; + bool truncated_status = false; if (!by_user) { usage_record_prefix_by_time(end, end_key); @@ -2922,11 +2935,14 @@ static int usage_iterate_range(cls_method_context_t hctx, uint64_t start, uint64 } CLS_LOG(20, "usage_iterate_range start_key=%s", start_key.c_str()); - int ret = cls_cxx_map_get_vals(hctx, start_key, filter_prefix, max_entries, &keys, truncated); + int ret = cls_cxx_map_get_vals(hctx, start_key, filter_prefix, max_entries, &keys, &truncated_status); if (ret < 0) return ret; - + if (truncated) { + *truncated = truncated_status; + } + map::iterator iter = keys.begin(); if (iter == keys.end()) return 0; @@ -2939,11 +2955,17 @@ static int usage_iterate_range(cls_method_context_t hctx, uint64_t start, uint64 if (!by_user && key.compare(end_key) >= 0) { CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str()); + if (truncated_status) { + key_iter = key; + } return 0; } if (by_user && key.compare(0, user_key.size(), user_key) != 0) { CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str()); + if (truncated_status) { + key_iter = key; + } return 0; } diff --git a/ceph/src/cls/user/cls_user.cc b/ceph/src/cls/user/cls_user.cc index 7912578aa..840470e9f 100644 --- a/ceph/src/cls/user/cls_user.cc +++ b/ceph/src/cls/user/cls_user.cc @@ -163,7 +163,6 @@ static int cls_user_set_buckets_info(cls_method_context_t hctx, bufferlist *in, if (!op.add){ apply_entry_stats(update_entry, &entry); } - entry.user_stats_sync = true; ret = write_entry(hctx, key, entry); diff --git a/ceph/src/cls/user/cls_user_types.h b/ceph/src/cls/user/cls_user_types.h index 8595f25fd..6ffd93323 100644 --- a/ceph/src/cls/user/cls_user_types.h +++ b/ceph/src/cls/user/cls_user_types.h @@ -101,14 +101,14 @@ struct cls_user_bucket_entry { cls_user_bucket bucket; size_t size; size_t size_rounded; - real_time creation_time; + ceph::real_time creation_time; uint64_t count; bool user_stats_sync; cls_user_bucket_entry() : size(0), size_rounded(0), count(0), user_stats_sync(false) {} void encode(bufferlist& bl) const { - ENCODE_START(7, 5, bl); + ENCODE_START(9, 5, bl); uint64_t s = size; __u32 mt = ceph::real_clock::to_time_t(creation_time); string empty_str; // originally had the bucket name here, but we encode bucket later @@ -121,10 +121,11 @@ struct cls_user_bucket_entry { ::encode(s, bl); ::encode(user_stats_sync, bl); ::encode(creation_time, bl); + //::encode(placement_rule, bl); removed in v9 ENCODE_FINISH(bl); } void decode(bufferlist::iterator& bl) { - DECODE_START_LEGACY_COMPAT_LEN(6, 5, 5, bl); + DECODE_START_LEGACY_COMPAT_LEN(9, 5, 5, bl); __u32 mt; uint64_t s; string empty_str; // backward compatibility @@ -146,6 +147,10 @@ struct cls_user_bucket_entry { ::decode(user_stats_sync, bl); if (struct_v >= 7) ::decode(creation_time, bl); + if (struct_v == 8) { // added in v8, removed in v9 + std::string placement_rule; + ::decode(placement_rule, bl); + } DECODE_FINISH(bl); } void dump(Formatter *f) const; diff --git a/ceph/src/common/AsyncReserver.h b/ceph/src/common/AsyncReserver.h index 28512ac80..d5c7a852d 100644 --- a/ceph/src/common/AsyncReserver.h +++ b/ceph/src/common/AsyncReserver.h @@ -18,6 +18,8 @@ #include "common/Finisher.h" #include "common/Formatter.h" +#define rdout(x) lgeneric_subdout(cct,reserver,x) + /** * Manages a configurable number of asyncronous reservations. * @@ -27,38 +29,104 @@ */ template class AsyncReserver { + CephContext *cct; Finisher *f; unsigned max_allowed; unsigned min_priority; Mutex lock; - map > > queues; - map >::iterator > > queue_pointers; - set in_progress; + struct Reservation { + T item; + unsigned prio = 0; + Context *grant = 0; + Context *preempt = 0; + Reservation() {} + Reservation(T i, unsigned pr, Context *g, Context *p = 0) + : item(i), prio(pr), grant(g), preempt(p) {} + void dump(Formatter *f) const { + f->dump_stream("item") << item; + f->dump_unsigned("prio", prio); + f->dump_bool("can_preempt", !!preempt); + } + friend ostream& operator<<(ostream& out, const Reservation& r) { + return out << r.item << "(prio " << r.prio << " grant " << r.grant + << " preempt " << r.preempt << ")"; + } + }; + + map> queues; + map::iterator>> queue_pointers; + map in_progress; + set> preempt_by_prio; ///< in_progress that can be preempted + + void preempt_one() { + assert(!preempt_by_prio.empty()); + auto q = in_progress.find(preempt_by_prio.begin()->second); + assert(q != in_progress.end()); + Reservation victim = q->second; + rdout(10) << __func__ << " preempt " << victim << dendl; + f->queue(victim.preempt); + victim.preempt = nullptr; + in_progress.erase(q); + preempt_by_prio.erase(preempt_by_prio.begin()); + } void do_queues() { - typename map > >::reverse_iterator it; - for (it = queues.rbegin(); - it != queues.rend() && - in_progress.size() < max_allowed && - it->first >= min_priority; - ++it) { - while (in_progress.size() < max_allowed && - !it->second.empty()) { - pair p = it->second.front(); - queue_pointers.erase(p.first); - it->second.pop_front(); - f->queue(p.second); - in_progress.insert(p.first); + rdout(20) << __func__ << ":\n"; + JSONFormatter jf(true); + jf.open_object_section("queue"); + _dump(&jf); + jf.close_section(); + jf.flush(*_dout); + *_dout << dendl; + + // in case min_priority was adjusted up or max_allowed was adjusted down + while (!preempt_by_prio.empty() && + (in_progress.size() > max_allowed || + preempt_by_prio.begin()->first < min_priority)) { + preempt_one(); + } + + while (!queues.empty()) { + // choose highest priority queue + auto it = queues.end(); + --it; + assert(!it->second.empty()); + if (it->first < min_priority) { + break; + } + if (in_progress.size() >= max_allowed && + !preempt_by_prio.empty() && + it->first > preempt_by_prio.begin()->first) { + preempt_one(); + } + if (in_progress.size() >= max_allowed) { + break; // no room + } + // grant + Reservation p = it->second.front(); + rdout(10) << __func__ << " grant " << p << dendl; + queue_pointers.erase(p.item); + it->second.pop_front(); + if (it->second.empty()) { + queues.erase(it); + } + f->queue(p.grant); + p.grant = nullptr; + in_progress[p.item] = p; + if (p.preempt) { + preempt_by_prio.insert(make_pair(p.prio, p.item)); } } } public: AsyncReserver( + CephContext *cct, Finisher *f, unsigned max_allowed, unsigned min_priority = 0) - : f(f), + : cct(cct), + f(f), max_allowed(max_allowed), min_priority(min_priority), lock("AsyncReserver::lock") {} @@ -77,27 +145,26 @@ public: void dump(Formatter *f) { Mutex::Locker l(lock); + _dump(f); + } + void _dump(Formatter *f) { f->dump_unsigned("max_allowed", max_allowed); f->dump_unsigned("min_priority", min_priority); f->open_array_section("queues"); - for (typename map > > ::const_iterator p = - queues.begin(); p != queues.end(); ++p) { + for (auto& p : queues) { f->open_object_section("queue"); - f->dump_unsigned("priority", p->first); + f->dump_unsigned("priority", p.first); f->open_array_section("items"); - for (typename list >::const_iterator q = - p->second.begin(); q != p->second.end(); ++q) { - f->dump_stream("item") << q->first; + for (auto& q : p.second) { + f->dump_object("item", q); } f->close_section(); f->close_section(); } f->close_section(); f->open_array_section("in_progress"); - for (typename set::const_iterator p = in_progress.begin(); - p != in_progress.end(); - ++p) { - f->dump_stream("item") << *p; + for (auto& p : in_progress) { + f->dump_object("item", p.second); } f->close_section(); } @@ -113,13 +180,17 @@ public: void request_reservation( T item, ///< [in] reservation key Context *on_reserved, ///< [in] callback to be called on reservation - unsigned prio + unsigned prio, ///< [in] priority + Context *on_preempt = 0 ///< [in] callback to be called if we are preempted (optional) ) { Mutex::Locker l(lock); + Reservation r(item, prio, on_reserved, on_preempt); + rdout(10) << __func__ << " queue " << r << dendl; assert(!queue_pointers.count(item) && !in_progress.count(item)); - queues[prio].push_back(make_pair(item, on_reserved)); - queue_pointers.insert(make_pair(item, make_pair(prio,--(queues[prio]).end()))); + queues[prio].push_back(r); + queue_pointers.insert(make_pair(item, + make_pair(prio,--(queues[prio]).end()))); do_queues(); } @@ -134,13 +205,31 @@ public: T item ///< [in] key for reservation to cancel ) { Mutex::Locker l(lock); - if (queue_pointers.count(item)) { - unsigned prio = queue_pointers[item].first; - delete queue_pointers[item].second->second; - queues[prio].erase(queue_pointers[item].second); - queue_pointers.erase(item); + auto i = queue_pointers.find(item); + if (i != queue_pointers.end()) { + unsigned prio = i->second.first; + const Reservation& r = *i->second.second; + rdout(10) << __func__ << " cancel " << r << " (was queued)" << dendl; + delete r.grant; + delete r.preempt; + queues[prio].erase(i->second.second); + if (queues[prio].empty()) { + queues.erase(prio); + } + queue_pointers.erase(i); } else { - in_progress.erase(item); + auto p = in_progress.find(item); + if (p != in_progress.end()) { + rdout(10) << __func__ << " cancel " << p->second + << " (was in progress)" << dendl; + if (p->second.preempt) { + preempt_by_prio.erase(make_pair(p->second.prio, p->second.item)); + delete p->second.preempt; + } + in_progress.erase(p); + } else { + rdout(10) << __func__ << " cancel " << item << " (not found)" << dendl; + } } do_queues(); } @@ -157,4 +246,5 @@ public: static const unsigned MAX_PRIORITY = (unsigned)-1; }; +#undef rdout #endif diff --git a/ceph/src/common/LogClient.cc b/ceph/src/common/LogClient.cc index aeb2f5bfc..6157194bb 100644 --- a/ceph/src/common/LogClient.cc +++ b/ceph/src/common/LogClient.cc @@ -88,7 +88,7 @@ int parse_log_client_options(CephContext *cct, return r; } - fsid = cct->_conf->fsid; + fsid = cct->_conf->get_val("fsid"); host = cct->_conf->host; return 0; } diff --git a/ceph/src/common/Timer.cc b/ceph/src/common/Timer.cc index f211a6f8f..45305f553 100644 --- a/ceph/src/common/Timer.cc +++ b/ceph/src/common/Timer.cc @@ -114,7 +114,7 @@ void SafeTimer::timer_thread() lock.Unlock(); } -bool SafeTimer::add_event_after(double seconds, Context *callback) +Context* SafeTimer::add_event_after(double seconds, Context *callback) { assert(lock.is_locked()); @@ -123,14 +123,14 @@ bool SafeTimer::add_event_after(double seconds, Context *callback) return add_event_at(when, callback); } -bool SafeTimer::add_event_at(utime_t when, Context *callback) +Context* SafeTimer::add_event_at(utime_t when, Context *callback) { assert(lock.is_locked()); ldout(cct,10) << __func__ << " " << when << " -> " << callback << dendl; if (stopping) { ldout(cct,5) << __func__ << " already shutdown, event not added" << dendl; delete callback; - return false; + return nullptr; } scheduled_map_t::value_type s_val(when, callback); scheduled_map_t::iterator i = schedule.insert(s_val); @@ -145,7 +145,7 @@ bool SafeTimer::add_event_at(utime_t when, Context *callback) * adjust our timeout. */ if (i == schedule.begin()) cond.Signal(); - return true; + return callback; } bool SafeTimer::cancel_event(Context *callback) diff --git a/ceph/src/common/Timer.h b/ceph/src/common/Timer.h index 861b239ca..8fd478a99 100644 --- a/ceph/src/common/Timer.h +++ b/ceph/src/common/Timer.h @@ -70,8 +70,8 @@ public: /* Schedule an event in the future * Call with the event_lock LOCKED */ - bool add_event_after(double seconds, Context *callback); - bool add_event_at(utime_t when, Context *callback); + Context* add_event_after(double seconds, Context *callback); + Context* add_event_at(utime_t when, Context *callback); /* Cancel an event. * Call with the event_lock LOCKED diff --git a/ceph/src/common/bit_vector.hpp b/ceph/src/common/bit_vector.hpp index 6a6e6b7d0..b010970b3 100644 --- a/ceph/src/common/bit_vector.hpp +++ b/ceph/src/common/bit_vector.hpp @@ -14,6 +14,7 @@ #include "common/Formatter.h" #include "include/assert.h" #include "include/encoding.h" +#include namespace ceph { @@ -28,36 +29,150 @@ private: // must be power of 2 BOOST_STATIC_ASSERT((_bit_count != 0) && !(_bit_count & (_bit_count - 1))); BOOST_STATIC_ASSERT(_bit_count <= BITS_PER_BYTE); -public: - static const uint32_t BLOCK_SIZE; - class ConstReference { + template + class ReferenceImpl { + protected: + DataIterator m_data_iterator; + uint64_t m_shift; + + ReferenceImpl(const DataIterator& data_iterator, uint64_t shift) + : m_data_iterator(data_iterator), m_shift(shift) { + } + ReferenceImpl(DataIterator&& data_iterator, uint64_t shift) + : m_data_iterator(std::move(data_iterator)), m_shift(shift) { + } + public: - operator uint8_t() const; + inline operator uint8_t() const { + return (*m_data_iterator >> m_shift) & MASK; + } + }; + +public: + + class ConstReference : public ReferenceImpl { private: friend class BitVector; - const BitVector &m_bit_vector; - uint64_t m_offset; - ConstReference(const BitVector &bit_vector, uint64_t offset); + ConstReference(const bufferlist::const_iterator& data_iterator, + uint64_t shift) + : ReferenceImpl(data_iterator, shift) { + } + ConstReference(bufferlist::const_iterator&& data_iterator, uint64_t shift) + : ReferenceImpl(std::move(data_iterator), + shift) { + } }; - class Reference { + class Reference : public ReferenceImpl { public: - operator uint8_t() const; Reference& operator=(uint8_t v); + + private: + friend class BitVector; + + Reference(const bufferlist::iterator& data_iterator, uint64_t shift) + : ReferenceImpl(data_iterator, shift) { + } + Reference(bufferlist::iterator&& data_iterator, uint64_t shift) + : ReferenceImpl(std::move(data_iterator), shift) { + } + }; + +public: + template + class IteratorImpl { private: friend class BitVector; - BitVector &m_bit_vector; - uint64_t m_offset; - Reference(BitVector &bit_vector, uint64_t offset); + uint64_t m_offset = 0; + BitVectorT *m_bit_vector; + + // cached derived values + uint64_t m_index = 0; + uint64_t m_shift = 0; + DataIterator m_data_iterator; + + IteratorImpl(BitVectorT *bit_vector, uint64_t offset) + : m_bit_vector(bit_vector), + m_data_iterator(bit_vector->m_data.begin()) { + *this += offset; + } + + public: + inline IteratorImpl& operator++() { + ++m_offset; + + uint64_t index; + compute_index(m_offset, &index, &m_shift); + + assert(index == m_index || index == m_index + 1); + if (index > m_index) { + m_index = index; + ++m_data_iterator; + } + return *this; + } + inline IteratorImpl& operator+=(uint64_t offset) { + m_offset += offset; + compute_index(m_offset, &m_index, &m_shift); + if (m_offset < m_bit_vector->size()) { + m_data_iterator.seek(m_index); + } else { + m_data_iterator = m_bit_vector->m_data.end(); + } + return *this; + } + + inline IteratorImpl operator++(int) { + IteratorImpl iterator_impl(*this); + ++iterator_impl; + return iterator_impl; + } + inline IteratorImpl operator+(uint64_t offset) { + IteratorImpl iterator_impl(*this); + iterator_impl += offset; + return iterator_impl; + } + + inline bool operator==(const IteratorImpl& rhs) const { + return (m_offset == rhs.m_offset && m_bit_vector == rhs.m_bit_vector); + } + inline bool operator!=(const IteratorImpl& rhs) const { + return (m_offset != rhs.m_offset || m_bit_vector != rhs.m_bit_vector); + } + + inline ConstReference operator*() const { + return ConstReference(m_data_iterator, m_shift); + } + inline Reference operator*() { + return Reference(m_data_iterator, m_shift); + } }; + typedef IteratorImpl ConstIterator; + typedef IteratorImpl Iterator; + + static const uint32_t BLOCK_SIZE; static const uint8_t BIT_COUNT = _bit_count; BitVector(); + inline ConstIterator begin() const { + return ConstIterator(this, 0); + } + inline ConstIterator end() const { + return ConstIterator(this, m_size); + } + inline Iterator begin() { + return Iterator(this, 0); + } + inline Iterator end() { + return Iterator(this, m_size); + } + void set_crc_enabled(bool enabled) { m_crc_enabled = enabled; } @@ -345,55 +460,33 @@ bool BitVector<_b>::operator==(const BitVector &b) const { template typename BitVector<_b>::Reference BitVector<_b>::operator[](uint64_t offset) { - return Reference(*this, offset); -} - -template -typename BitVector<_b>::ConstReference BitVector<_b>::operator[](uint64_t offset) const { - return ConstReference(*this, offset); -} - -template -BitVector<_b>::ConstReference::ConstReference(const BitVector<_b> &bit_vector, - uint64_t offset) - : m_bit_vector(bit_vector), m_offset(offset) -{ -} - -template -BitVector<_b>::ConstReference::operator uint8_t() const { uint64_t index; uint64_t shift; - this->m_bit_vector.compute_index(this->m_offset, &index, &shift); + compute_index(offset, &index, &shift); - return (this->m_bit_vector.m_data[index] >> shift) & MASK; + bufferlist::iterator data_iterator(m_data.begin()); + data_iterator.seek(index); + return Reference(std::move(data_iterator), shift); } template -BitVector<_b>::Reference::Reference(BitVector<_b> &bit_vector, uint64_t offset) - : m_bit_vector(bit_vector), m_offset(offset) -{ -} - -template -BitVector<_b>::Reference::operator uint8_t() const { +typename BitVector<_b>::ConstReference BitVector<_b>::operator[](uint64_t offset) const { uint64_t index; uint64_t shift; - this->m_bit_vector.compute_index(this->m_offset, &index, &shift); + compute_index(offset, &index, &shift); - return (this->m_bit_vector.m_data[index] >> shift) & MASK; + bufferlist::const_iterator data_iterator(m_data.begin()); + data_iterator.seek(index); + return ConstReference(std::move(data_iterator), shift); } template typename BitVector<_b>::Reference& BitVector<_b>::Reference::operator=(uint8_t v) { - uint64_t index; - uint64_t shift; - this->m_bit_vector.compute_index(this->m_offset, &index, &shift); - - uint8_t mask = MASK << shift; - char packed_value = (this->m_bit_vector.m_data[index] & ~mask) | - ((v << shift) & mask); - this->m_bit_vector.m_data.copy_in(index, 1, &packed_value); + uint8_t mask = MASK << this->m_shift; + char packed_value = (*this->m_data_iterator & ~mask) | + ((v << this->m_shift) & mask); + bufferlist::iterator it(this->m_data_iterator); + it.copy_in(1, &packed_value, true); return *this; } diff --git a/ceph/src/common/buffer.cc b/ceph/src/common/buffer.cc index b8e87d1ee..18ae276cc 100644 --- a/ceph/src/common/buffer.cc +++ b/ceph/src/common/buffer.cc @@ -172,17 +172,17 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT; char *data; unsigned len; std::atomic nref { 0 }; - int mempool = mempool::mempool_buffer_anon; + int mempool; mutable std::atomic_flag crc_spinlock = ATOMIC_FLAG_INIT; map, pair > crc_map; - explicit raw(unsigned l) - : data(NULL), len(l), nref(0) { + explicit raw(unsigned l, int mempool=mempool::mempool_buffer_anon) + : data(NULL), len(l), nref(0), mempool(mempool) { mempool::get_pool(mempool::pool_index_t(mempool)).adjust_count(1, len); } - raw(char *c, unsigned l) - : data(c), len(l), nref(0) { + raw(char *c, unsigned l, int mempool=mempool::mempool_buffer_anon) + : data(c), len(l), nref(0), mempool(mempool) { mempool::get_pool(mempool::pool_index_t(mempool)).adjust_count(1, len); } virtual ~raw() { @@ -281,8 +281,9 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT; class buffer::raw_combined : public buffer::raw { size_t alignment; public: - raw_combined(char *dataptr, unsigned l, unsigned align=0) - : raw(dataptr, l), + raw_combined(char *dataptr, unsigned l, unsigned align, + int mempool) + : raw(dataptr, l, mempool), alignment(align) { inc_total_alloc(len); inc_history_alloc(len); @@ -294,7 +295,9 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT; return create(len, alignment); } - static raw_combined *create(unsigned len, unsigned align=0) { + static raw_combined *create(unsigned len, + unsigned align, + int mempool = mempool::mempool_buffer_anon) { if (!align) align = sizeof(size_t); size_t rawlen = ROUND_UP_TO(sizeof(buffer::raw_combined), @@ -314,7 +317,7 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT; // actual data first, since it has presumably larger alignment restriction // then put the raw_combined at the end - return new (ptr + datalen) raw_combined(ptr, len, align); + return new (ptr + datalen) raw_combined(ptr, len, align, mempool); } static void operator delete(void *ptr) { @@ -771,6 +774,9 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT; buffer::raw* buffer::create(unsigned len) { return buffer::create_aligned(len, sizeof(size_t)); } + buffer::raw* buffer::create_in_mempool(unsigned len, int mempool) { + return buffer::create_aligned_in_mempool(len, sizeof(size_t), mempool); + } buffer::raw* buffer::claim_char(unsigned len, char *buf) { return new raw_claimed_char(len, buf); } @@ -787,7 +793,8 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT; return new raw_claim_buffer(buf, len, std::move(del)); } - buffer::raw* buffer::create_aligned(unsigned len, unsigned align) { + buffer::raw* buffer::create_aligned_in_mempool( + unsigned len, unsigned align, int mempool) { // If alignment is a page multiple, use a separate buffer::raw to // avoid fragmenting the heap. // @@ -805,7 +812,12 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT; return new raw_hack_aligned(len, align); #endif } - return raw_combined::create(len, align); + return raw_combined::create(len, align, mempool); + } + buffer::raw* buffer::create_aligned( + unsigned len, unsigned align) { + return create_aligned_in_mempool(len, align, + mempool::mempool_buffer_anon); } buffer::raw* buffer::create_page_aligned(unsigned len) { @@ -952,6 +964,24 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT; bool buffer::ptr::at_buffer_tail() const { return _off + _len == _raw->len; } + int buffer::ptr::get_mempool() const { + if (_raw) { + return _raw->mempool; + } + return mempool::mempool_buffer_anon; + } + + void buffer::ptr::reassign_to_mempool(int pool) { + if (_raw) { + _raw->reassign_to_mempool(pool); + } + } + void buffer::ptr::try_assign_to_mempool(int pool) { + if (_raw) { + _raw->try_assign_to_mempool(pool); + } + } + const char *buffer::ptr::c_str() const { assert(_raw); if (buffer_track_c_str) @@ -1493,7 +1523,6 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT; { std::swap(_len, other._len); std::swap(_memcopy_count, other._memcopy_count); - std::swap(_mempool, other._mempool); _buffers.swap(other._buffers); append_buffer.swap(other.append_buffer); //last_p.swap(other.last_p); @@ -1666,9 +1695,16 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT; return is_aligned(CEPH_PAGE_SIZE); } + int buffer::list::get_mempool() const + { + if (_buffers.empty()) { + return mempool::mempool_buffer_anon; + } + return _buffers.back().get_mempool(); + } + void buffer::list::reassign_to_mempool(int pool) { - _mempool = pool; if (append_buffer.get_raw()) { append_buffer.get_raw()->reassign_to_mempool(pool); } @@ -1679,7 +1715,6 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT; void buffer::list::try_assign_to_mempool(int pool) { - _mempool = pool; if (append_buffer.get_raw()) { append_buffer.get_raw()->try_assign_to_mempool(pool); } @@ -1778,10 +1813,7 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT; void buffer::list::reserve(size_t prealloc) { if (append_buffer.unused_tail_length() < prealloc) { - append_buffer = buffer::create(prealloc); - if (_mempool >= 0) { - append_buffer.get_raw()->reassign_to_mempool(_mempool); - } + append_buffer = buffer::create_in_mempool(prealloc, get_mempool()); append_buffer.set_length(0); // unused, so far. } } @@ -1879,11 +1911,9 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT; unsigned gap = append_buffer.unused_tail_length(); if (!gap) { // make a new append_buffer! - append_buffer = raw_combined::create(CEPH_BUFFER_APPEND_SIZE); + append_buffer = raw_combined::create(CEPH_BUFFER_APPEND_SIZE, 0, + get_mempool()); append_buffer.set_length(0); // unused, so far. - if (_mempool >= 0) { - append_buffer.get_raw()->reassign_to_mempool(_mempool); - } } append(append_buffer, append_buffer.append(c) - 1, 1); // add segment to the list } @@ -1909,11 +1939,8 @@ static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT; size_t need = ROUND_UP_TO(len, sizeof(size_t)) + sizeof(raw_combined); size_t alen = ROUND_UP_TO(need, CEPH_BUFFER_ALLOC_UNIT) - sizeof(raw_combined); - append_buffer = raw_combined::create(alen); + append_buffer = raw_combined::create(alen, 0, get_mempool()); append_buffer.set_length(0); // unused, so far. - if (_mempool >= 0) { - append_buffer.get_raw()->reassign_to_mempool(_mempool); - } } } diff --git a/ceph/src/common/ceph_context.cc b/ceph/src/common/ceph_context.cc index 2cf0f7d8c..64b2ec674 100644 --- a/ceph/src/common/ceph_context.cc +++ b/ceph/src/common/ceph_context.cc @@ -259,7 +259,7 @@ public: } if (log->graylog() && changed.count("fsid")) { - log->graylog()->set_fsid(conf->fsid); + log->graylog()->set_fsid(conf->get_val("fsid")); } } }; diff --git a/ceph/src/common/common_init.cc b/ceph/src/common/common_init.cc index 9cb1b1207..7889f42a0 100644 --- a/ceph/src/common/common_init.cc +++ b/ceph/src/common/common_init.cc @@ -58,6 +58,10 @@ CephContext *common_preinit(const CephInitParameters &iparams, conf->set_val_or_die("err_to_stderr", "false"); conf->set_val_or_die("log_flush_on_exit", "false"); } + if (code_env != CODE_ENVIRONMENT_DAEMON) { + // NOTE: disable ms subsystem gathering in clients by default + conf->set_val_or_die("debug_ms", "0/0"); + } return cct; } diff --git a/ceph/src/common/config.cc b/ceph/src/common/config.cc index ea372bfb3..3cbb27e34 100644 --- a/ceph/src/common/config.cc +++ b/ceph/src/common/config.cc @@ -492,7 +492,10 @@ int md_config_t::parse_argv(std::vector& args) set_val_or_die("client_mountpoint", val.c_str()); } else { - parse_option(args, i, NULL); + int r = parse_option(args, i, NULL); + if (r < 0) { + return r; + } } } @@ -536,8 +539,16 @@ int md_config_t::parse_option(std::vector& args, std::string as_option("--"); as_option += "debug_"; as_option += subsys.get_name(o); - if (ceph_argparse_witharg(args, i, &val, + ostringstream err; + if (ceph_argparse_witharg(args, i, &val, err, as_option.c_str(), (char*)NULL)) { + if (err.tellp()) { + if (oss) { + *oss << err.str(); + } + ret = -EINVAL; + break; + } int log, gather; int r = sscanf(val.c_str(), "%d/%d", &log, &gather); if (r >= 1) { diff --git a/ceph/src/common/legacy_config_opts.h b/ceph/src/common/legacy_config_opts.h index cb6b406bb..e0f0ad7e3 100644 --- a/ceph/src/common/legacy_config_opts.h +++ b/ceph/src/common/legacy_config_opts.h @@ -14,15 +14,11 @@ /* note: no header guard */ OPTION(host, OPT_STR) // "" means that ceph will use short hostname -OPTION(fsid, OPT_UUID) OPTION(public_addr, OPT_ADDR) OPTION(public_bind_addr, OPT_ADDR) OPTION(cluster_addr, OPT_ADDR) OPTION(public_network, OPT_STR) OPTION(cluster_network, OPT_STR) -OPTION(monmap, OPT_STR) -OPTION(mon_host, OPT_STR) -OPTION(mon_dns_srv_name, OPT_STR) OPTION(lockdep, OPT_BOOL) OPTION(lockdep_force_backtrace, OPT_BOOL) // always gather current backtrace at every lock OPTION(run_dir, OPT_STR) // the "/var/run/ceph" dir, created on daemon startup @@ -239,8 +235,6 @@ OPTION(mon_timecheck_interval, OPT_FLOAT) // on leader, timecheck (clock drift c OPTION(mon_timecheck_skew_interval, OPT_FLOAT) // on leader, timecheck (clock drift check) interval when in presence of a skew (seconds) OPTION(mon_pg_stuck_threshold, OPT_INT) // number of seconds after which pgs can be considered stuck inactive, unclean, etc (see doc/control.rst under dump_stuck for more info) OPTION(mon_pg_min_inactive, OPT_U64) // the number of PGs which have to be inactive longer than 'mon_pg_stuck_threshold' before health goes into ERR. 0 means disabled, never go into ERR. -OPTION(mon_pg_warn_min_per_osd, OPT_INT) // min # pgs per (in) osd before we warn the admin -OPTION(mon_pg_warn_max_per_osd, OPT_INT) // max # pgs per (in) osd before we warn the admin OPTION(mon_pg_warn_max_object_skew, OPT_FLOAT) // max skew few average in objects per pg OPTION(mon_pg_warn_min_objects, OPT_INT) // do not warn below this object # OPTION(mon_pg_warn_min_pool_objects, OPT_INT) // do not warn on pools below this object # @@ -267,7 +261,6 @@ OPTION(mon_max_mdsmap_epochs, OPT_INT) OPTION(mon_max_osd, OPT_INT) OPTION(mon_probe_timeout, OPT_DOUBLE) OPTION(mon_client_bytes, OPT_U64) // client msg data allowed in memory (in bytes) -OPTION(mon_mgr_proxy_client_bytes_ratio, OPT_FLOAT) // ratio of mon_client_bytes that can be consumed by proxied mgr commands before we error out to client OPTION(mon_log_max_summary, OPT_U64) OPTION(mon_daemon_bytes, OPT_U64) // mds, osd message memory cap (in bytes) OPTION(mon_max_log_entries_per_event, OPT_INT) @@ -1541,27 +1534,6 @@ OPTION(rgw_shard_warning_threshold, OPT_DOUBLE) // pct of safe max OPTION(rgw_swift_versioning_enabled, OPT_BOOL) // whether swift object versioning feature is enabled -OPTION(mgr_module_path, OPT_STR) // where to load python modules from -OPTION(mgr_initial_modules, OPT_STR) // Which modules to load -OPTION(mgr_data, OPT_STR) // where to find keyring etc -OPTION(mgr_tick_period, OPT_INT) // How frequently to tick -OPTION(mgr_stats_period, OPT_INT) // How frequently clients send stats -OPTION(mgr_client_bytes, OPT_U64) // bytes from clients -OPTION(mgr_client_messages, OPT_U64) // messages from clients -OPTION(mgr_osd_bytes, OPT_U64) // bytes from osds -OPTION(mgr_osd_messages, OPT_U64) // messages from osds -OPTION(mgr_mds_bytes, OPT_U64) // bytes from mdss -OPTION(mgr_mds_messages, OPT_U64) // messages from mdss -OPTION(mgr_mon_bytes, OPT_U64) // bytes from mons -OPTION(mgr_mon_messages, OPT_U64) // messages from mons - -OPTION(mgr_connect_retry_interval, OPT_DOUBLE) -OPTION(mgr_service_beacon_grace, OPT_DOUBLE) - -OPTION(mon_mgr_digest_period, OPT_INT) // How frequently to send digests -OPTION(mon_mgr_beacon_grace, OPT_INT) // How long to wait to failover -OPTION(mon_mgr_inactive_grace, OPT_INT) // How long before health WARN -> ERR -OPTION(mon_mgr_mkfs_grace, OPT_INT) // How long before we complain about MGR_DOWN OPTION(rgw_crypt_require_ssl, OPT_BOOL) // requests including encryption key headers must be sent over ssl OPTION(rgw_crypt_default_encryption_key, OPT_STR) // base64 encoded key for encryption of rgw objects OPTION(rgw_crypt_s3_kms_encryption_keys, OPT_STR) // extra keys that may be used for aws:kms diff --git a/ceph/src/common/options.cc b/ceph/src/common/options.cc index 0be052e1b..33f920525 100644 --- a/ceph/src/common/options.cc +++ b/ceph/src/common/options.cc @@ -11,6 +11,9 @@ #include #include +// Definitions for enums +#include "common/perf_counters.h" + void Option::dump_value(const char *field_name, const Option::value_t &v, Formatter *f) const @@ -160,12 +163,24 @@ std::vector