From 35e4c4457ede68661c55bb19d5cb7acb7f8b39eb Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fabian=20Gr=C3=BCnbichler?= Date: Wed, 16 Aug 2017 09:18:31 +0200 Subject: [PATCH] update sources to v12.1.4 --- ceph/CMakeLists.txt | 2 +- ceph/alpine/APKBUILD | 6 +- ceph/ceph.spec | 6 +- ceph/debian/changelog | 6 + ceph/doc/man/8/ceph.rst | 24 ++- ceph/doc/release-notes.rst | 5 + ceph/qa/standalone/crush/crush-classes.sh | 12 ++ .../multimon/tasks/mon_clock_with_skews.yaml | 5 +- .../rados/verify/validater/valgrind.yaml | 4 + ceph/qa/tasks/cephfs/filesystem.py | 13 ++ ceph/qa/tasks/thrashosds-health.yaml | 1 + ceph/qa/workunits/mon/crush_ops.sh | 31 ++++ ceph/src/.git_version | 4 +- ceph/src/common/options.cc | 2 +- ceph/src/crush/CrushWrapper.cc | 151 ++++++++++++++--- ceph/src/crush/CrushWrapper.h | 9 +- ceph/src/crush/builder.c | 18 +- ceph/src/mgr/DaemonServer.cc | 157 ++++++++++++++++++ ceph/src/mgr/MgrCommands.h | 7 + ceph/src/mon/FSCommands.cc | 25 +-- ceph/src/mon/MonCommands.h | 5 + ceph/src/mon/Monitor.cc | 2 +- ceph/src/mon/OSDMonitor.cc | 51 +++++- ceph/src/mon/Paxos.cc | 23 +++ ceph/src/mon/Paxos.h | 11 ++ ceph/src/mon/PaxosService.h | 11 +- ceph/src/os/bluestore/BlueStore.cc | 57 +++++-- ceph/src/os/bluestore/BlueStore.h | 2 +- ceph/src/osd/OSD.cc | 6 +- ceph/src/osd/OSD.h | 8 +- ceph/src/osd/OSDMap.cc | 20 +++ ceph/src/osd/OSDMap.h | 4 + ceph/src/osd/PGLog.h | 9 +- ceph/src/osd/osd_types.cc | 7 +- ceph/src/osd/osd_types.h | 7 +- ceph/src/test/crush/CrushWrapper.cc | 27 ++- ceph/src/test/objectstore/store_test.cc | 4 +- ceph/src/test/osd/TestOSDMap.cc | 39 +++++ ceph/src/tools/rados/rados.cc | 2 +- 39 files changed, 681 insertions(+), 102 deletions(-) diff --git a/ceph/CMakeLists.txt b/ceph/CMakeLists.txt index 0aaf7dbc6..bc7dd87ee 100644 --- a/ceph/CMakeLists.txt +++ b/ceph/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 2.8.11) project(ceph) -set(VERSION 12.1.3) +set(VERSION 12.1.4) if(POLICY CMP0046) # Tweak policies (this one disables "missing" dependency warning) diff --git a/ceph/alpine/APKBUILD b/ceph/alpine/APKBUILD index c3a300c88..48999f82c 100644 --- a/ceph/alpine/APKBUILD +++ b/ceph/alpine/APKBUILD @@ -1,7 +1,7 @@ # Contributor: John Coyle # Maintainer: John Coyle pkgname=ceph -pkgver=12.1.3 +pkgver=12.1.4 pkgrel=0 pkgdesc="Ceph is a distributed object store and file system" pkgusers="ceph" @@ -63,7 +63,7 @@ makedepends=" xmlstarlet yasm " -source="ceph-12.1.3.tar.bz2" +source="ceph-12.1.4.tar.bz2" subpackages=" $pkgname-base $pkgname-common @@ -116,7 +116,7 @@ _sysconfdir=/etc _udevrulesdir=/etc/udev/rules.d _python_sitelib=/usr/lib/python2.7/site-packages -builddir=$srcdir/ceph-12.1.3 +builddir=$srcdir/ceph-12.1.4 build() { export CEPH_BUILD_VIRTUALENV=$builddir diff --git a/ceph/ceph.spec b/ceph/ceph.spec index 8473b8ac0..cd0076237 100644 --- a/ceph/ceph.spec +++ b/ceph/ceph.spec @@ -61,7 +61,7 @@ # main package definition ################################################################################# Name: ceph -Version: 12.1.3 +Version: 12.1.4 Release: 0%{?dist} %if 0%{?fedora} || 0%{?rhel} Epoch: 2 @@ -76,7 +76,7 @@ License: LGPL-2.1 and CC-BY-SA-1.0 and GPL-2.0 and BSL-1.0 and BSD-3-Clause and Group: System/Filesystems %endif URL: http://ceph.com/ -Source0: http://ceph.com/download/ceph-12.1.3.tar.bz2 +Source0: http://ceph.com/download/ceph-12.1.4.tar.bz2 %if 0%{?suse_version} %if 0%{?is_opensuse} ExclusiveArch: x86_64 aarch64 ppc64 ppc64le @@ -772,7 +772,7 @@ python-rbd, python-rgw or python-cephfs instead. # common ################################################################################# %prep -%autosetup -p1 -n ceph-12.1.3 +%autosetup -p1 -n ceph-12.1.4 %build %if 0%{with cephfs_java} diff --git a/ceph/debian/changelog b/ceph/debian/changelog index 481ec56e4..f6ac9bf92 100644 --- a/ceph/debian/changelog +++ b/ceph/debian/changelog @@ -1,3 +1,9 @@ +ceph (12.1.4-1) stable; urgency=medium + + * New upstream release + + -- Ceph Release Team Tue, 15 Aug 2017 13:45:08 +0000 + ceph (12.1.3-1) stable; urgency=medium * New upstream release diff --git a/ceph/doc/man/8/ceph.rst b/ceph/doc/man/8/ceph.rst index 0f372d971..a95b4a7e2 100644 --- a/ceph/doc/man/8/ceph.rst +++ b/ceph/doc/man/8/ceph.rst @@ -39,7 +39,7 @@ Synopsis | **ceph** **mon_status** -| **ceph** **osd** [ *blacklist* \| *blocked-by* \| *create* \| *new* \| *deep-scrub* \| *df* \| *down* \| *dump* \| *erasure-code-profile* \| *find* \| *getcrushmap* \| *getmap* \| *getmaxosd* \| *in* \| *lspools* \| *map* \| *metadata* \| *out* \| *pause* \| *perf* \| *pg-temp* \| *force-create-pg* \| *primary-affinity* \| *primary-temp* \| *repair* \| *reweight* \| *reweight-by-pg* \| *rm* \| *destroy* \| *purge* \| *scrub* \| *set* \| *setcrushmap* \| *setmaxosd* \| *stat* \| *tree* \| *unpause* \| *unset* ] ... +| **ceph** **osd** [ *blacklist* \| *blocked-by* \| *create* \| *new* \| *deep-scrub* \| *df* \| *down* \| *dump* \| *erasure-code-profile* \| *find* \| *getcrushmap* \| *getmap* \| *getmaxosd* \| *in* \| *lspools* \| *map* \| *metadata* \| *ok-to-stop* \| *out* \| *pause* \| *perf* \| *pg-temp* \| *force-create-pg* \| *primary-affinity* \| *primary-temp* \| *repair* \| *reweight* \| *reweight-by-pg* \| *rm* \| *destroy* \| *purge* \| *safe-to-destroy* \| *scrub* \| *set* \| *setcrushmap* \| *setmaxosd* \| *stat* \| *tree* \| *unpause* \| *unset* ] ... | **ceph** **osd** **crush** [ *add* \| *add-bucket* \| *create-or-move* \| *dump* \| *get-tunable* \| *link* \| *move* \| *remove* \| *rename-bucket* \| *reweight* \| *reweight-all* \| *reweight-subtree* \| *rm* \| *rule* \| *set* \| *set-tunable* \| *show-tunables* \| *tunables* \| *unlink* ] ... @@ -874,6 +874,18 @@ Usage:: ceph osd out [...] +Subcommand ``ok-to-stop`` checks whether the list of OSD(s) can be +stopped without immediately making data unavailable. That is, all +data should remain readable and writeable, although data redundancy +may be reduced as some PGs may end up in a degraded (but active) +state. It will return a success code if it is okay to stop the +OSD(s), or an error code and informative message if it is not or if no +conclusion can be drawn at the current time. + +Usage:: + + ceph osd ok-to-stop [...] + Subcommand ``pause`` pauses osd. Usage:: @@ -1066,6 +1078,16 @@ Usage:: ceph osd purge {--yes-i-really-mean-it} +Subcommand ``safe-to-destroy`` checks whether it is safe to remove or +destroy an OSD without reducing overall data redundancy or durability. +It will return a success code if it is definitely safe, or an error +code and informative message if it is not or if no conclusion can be +drawn at the current time. + +Usage:: + + ceph osd safe-to-destroy [...] + Subcommand ``scrub`` initiates scrub on specified osd. Usage:: diff --git a/ceph/doc/release-notes.rst b/ceph/doc/release-notes.rst index db0dbbc8b..76a897780 100644 --- a/ceph/doc/release-notes.rst +++ b/ceph/doc/release-notes.rst @@ -210,6 +210,11 @@ Major Changes from Kraken - ``ceph osd {add,rm}-{noout,noin,nodown,noup}`` allow the `noout`, `noin`, `nodown`, and `noup` flags to be applied to specific OSDs. + - ``ceph osd safe-to-destroy `` will report whether it is safe to + remove or destroy OSD(s) without reducing data durability or redundancy. + - ``ceph osd ok-to-stop `` will report whether it is okay to stop + OSD(s) without immediately compromising availability (i.e., all PGs + should remain active but may be degraded). - ``ceph log last [n]`` will output the last *n* lines of the cluster log. - ``ceph mgr dump`` will dump the MgrMap, including the currently active diff --git a/ceph/qa/standalone/crush/crush-classes.sh b/ceph/qa/standalone/crush/crush-classes.sh index 24696fcd3..160f9d8c1 100755 --- a/ceph/qa/standalone/crush/crush-classes.sh +++ b/ceph/qa/standalone/crush/crush-classes.sh @@ -201,6 +201,18 @@ function TEST_mon_classes() { # test set-device-class implicitly change class ceph osd crush set-device-class hdd osd.0 || return 1 expect_failure $dir EBUSY ceph osd crush set-device-class nvme osd.0 || return 1 + + # test class rename + ceph osd crush rm-device-class all || return 1 + ceph osd crush set-device-class class_1 all || return 1 + ceph osd crush class ls | grep 'class_1' || return 1 + ceph osd crush tree --show-shadow | grep 'class_1' || return 1 + ceph osd crush rule create-replicated class_1_rule default host class_1 || return 1 + ceph osd crush class rename class_1 class_2 + ceph osd crush class ls | grep 'class_1' && return 1 + ceph osd crush tree --show-shadow | grep 'class_1' && return 1 + ceph osd crush class ls | grep 'class_2' || return 1 + ceph osd crush tree --show-shadow | grep 'class_2' || return 1 } main crush-classes "$@" diff --git a/ceph/qa/suites/rados/multimon/tasks/mon_clock_with_skews.yaml b/ceph/qa/suites/rados/multimon/tasks/mon_clock_with_skews.yaml index 2bba60715..a2d3d0b65 100644 --- a/ceph/qa/suites/rados/multimon/tasks/mon_clock_with_skews.yaml +++ b/ceph/qa/suites/rados/multimon/tasks/mon_clock_with_skews.yaml @@ -10,6 +10,9 @@ tasks: - .*clock.*skew.* - clocks not synchronized - overall HEALTH_ - - (MON_CLOCK_SKEW) + - \(MON_CLOCK_SKEW\) + - \(MGR_DOWN\) + - \(PG_ + - No standby daemons available - mon_clock_skew_check: expect-skew: true diff --git a/ceph/qa/suites/rados/verify/validater/valgrind.yaml b/ceph/qa/suites/rados/verify/validater/valgrind.yaml index b2095b0c9..56224149f 100644 --- a/ceph/qa/suites/rados/verify/validater/valgrind.yaml +++ b/ceph/qa/suites/rados/verify/validater/valgrind.yaml @@ -12,6 +12,10 @@ overrides: osd heartbeat grace: 40 mon: mon osd crush smoke test: false + log-whitelist: + - overall HEALTH_ +# valgrind is slow.. we might get PGs stuck peering etc + - \(PG_ valgrind: mon: [--tool=memcheck, --leak-check=full, --show-reachable=yes] osd: [--tool=memcheck] diff --git a/ceph/qa/tasks/cephfs/filesystem.py b/ceph/qa/tasks/cephfs/filesystem.py index 0d3fc547b..dad3bd87b 100644 --- a/ceph/qa/tasks/cephfs/filesystem.py +++ b/ceph/qa/tasks/cephfs/filesystem.py @@ -453,6 +453,8 @@ class Filesystem(MDSCluster): data_pool_name, pgs_per_fs_pool.__str__()) self.mon_manager.raw_cluster_cmd('fs', 'new', self.name, self.metadata_pool_name, data_pool_name) + self.check_pool_application(self.metadata_pool_name) + self.check_pool_application(data_pool_name) # Turn off spurious standby count warnings from modifying max_mds in tests. try: self.mon_manager.raw_cluster_cmd('fs', 'set', self.name, 'standby_count_wanted', '0') @@ -465,6 +467,17 @@ class Filesystem(MDSCluster): self.getinfo(refresh = True) + + def check_pool_application(self, pool_name): + osd_map = self.mon_manager.get_osd_dump_json() + for pool in osd_map['pools']: + if pool['pool_name'] == pool_name: + if "application_metadata" in pool: + if not "cephfs" in pool['application_metadata']: + raise RuntimeError("Pool %p does not name cephfs as application!".\ + format(pool_name)) + + def __del__(self): if getattr(self._ctx, "filesystem", None) == self: delattr(self._ctx, "filesystem") diff --git a/ceph/qa/tasks/thrashosds-health.yaml b/ceph/qa/tasks/thrashosds-health.yaml index 3c9c4d226..9defe69ef 100644 --- a/ceph/qa/tasks/thrashosds-health.yaml +++ b/ceph/qa/tasks/thrashosds-health.yaml @@ -11,3 +11,4 @@ overrides: - \(OBJECT_ - \(REQUEST_SLOW\) - \(TOO_FEW_PGS\) + - \(MON_DOWN\) diff --git a/ceph/qa/workunits/mon/crush_ops.sh b/ceph/qa/workunits/mon/crush_ops.sh index a9f1512c5..1ef6e5cc1 100755 --- a/ceph/qa/workunits/mon/crush_ops.sh +++ b/ceph/qa/workunits/mon/crush_ops.sh @@ -155,4 +155,35 @@ ceph osd crush weight-set create-compat ceph osd crush weight-set ls | grep '(compat)' ceph osd crush weight-set rm-compat +# weight set vs device classes +ceph osd pool create cool 2 +ceph osd pool create cold 2 +ceph osd pool set cold size 2 +ceph osd crush weight-set create-compat +ceph osd crush weight-set create cool flat +ceph osd crush weight-set create cold positional +ceph osd crush rm-device-class osd.0 +ceph osd crush weight-set reweight-compat osd.0 10.5 +ceph osd crush weight-set reweight cool osd.0 11.5 +ceph osd crush weight-set reweight cold osd.0 12.5 12.4 +ceph osd crush set-device-class fish osd.0 +ceph osd crush tree --show-shadow | grep osd\\.0 | grep fish | grep 10\\. +ceph osd crush tree --show-shadow | grep osd\\.0 | grep fish | grep 11\\. +ceph osd crush tree --show-shadow | grep osd\\.0 | grep fish | grep 12\\. +ceph osd crush rm-device-class osd.0 +ceph osd crush set-device-class globster osd.0 +ceph osd crush tree --show-shadow | grep osd\\.0 | grep globster | grep 10\\. +ceph osd crush tree --show-shadow | grep osd\\.0 | grep globster | grep 11\\. +ceph osd crush tree --show-shadow | grep osd\\.0 | grep globster | grep 12\\. +ceph osd crush weight-set reweight-compat osd.0 7.5 +ceph osd crush weight-set reweight cool osd.0 8.5 +ceph osd crush weight-set reweight cold osd.0 6.5 6.6 +ceph osd crush tree --show-shadow | grep osd\\.0 | grep globster | grep 7\\. +ceph osd crush tree --show-shadow | grep osd\\.0 | grep globster | grep 8\\. +ceph osd crush tree --show-shadow | grep osd\\.0 | grep globster | grep 6\\. +ceph osd crush rm-device-class osd.0 +ceph osd pool rm cool cool --yes-i-really-really-mean-it +ceph osd pool rm cold cold --yes-i-really-really-mean-it +ceph osd crush weight-set rm-compat + echo OK diff --git a/ceph/src/.git_version b/ceph/src/.git_version index 5d44bb535..a0db25312 100644 --- a/ceph/src/.git_version +++ b/ceph/src/.git_version @@ -1,2 +1,2 @@ -c56d9c07b342c08419bbc18dcf2a4c5fae62b9cf -v12.1.3 +a5f84b37668fc8e03165aaf5cbb380c78e4deba4 +v12.1.4 diff --git a/ceph/src/common/options.cc b/ceph/src/common/options.cc index cfb518d77..2515d6aaa 100644 --- a/ceph/src/common/options.cc +++ b/ceph/src/common/options.cc @@ -2982,7 +2982,7 @@ std::vector