3 # Copyright (C) 2013,2014 Cloudwatt <libre.licensing@cloudwatt.com>
4 # Copyright (C) 2014,2015 Red Hat <contact@redhat.com>
5 # Copyright (C) 2014 Federico Gimenez <fgimenez@coit.es>
7 # Author: Loic Dachary <loic@dachary.org>
8 # Author: Federico Gimenez <fgimenez@coit.es>
10 # This program is free software; you can redistribute it and/or modify
11 # it under the terms of the GNU Library Public License as published by
12 # the Free Software Foundation; either version 2, or (at your option)
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU Library Public License for more details.
22 : ${CEPH_BUILD_VIRTUALENV:=/tmp}
24 if type xmlstarlet
> /dev
/null
2>&1; then
26 elif type xml
> /dev
/null
2>&1; then
29 echo "Missing xmlstarlet binary!"
33 if [ `uname` = FreeBSD
]; then
38 termwidth
=$
(stty
-a |
head -1 |
sed -e 's/.*columns \([0-9]*\).*/\1/')
39 if [ -n "$termwidth" -a "$termwidth" != "0" ]; then
40 termwidth
="-W ${termwidth}"
42 DIFFCOLOPTS
="-y $termwidth"
45 #! @file ceph-helpers.sh
46 # @brief Toolbox to manage Ceph cluster dedicated to testing
50 # ~~~~~~~~~~~~~~~~{.sh}
51 # source ceph-helpers.sh
54 # # cleanup leftovers and reset mydir
56 # # create a cluster with one monitor and three osds
61 # # put and get an object
62 # rados --pool rbd put GROUP /etc/group
63 # rados --pool rbd get GROUP /tmp/GROUP
64 # # stop the cluster and cleanup the directory
69 # The focus is on simplicity and efficiency, in the context of
70 # functional tests. The output is intentionally very verbose
71 # and functions return as soon as an error is found. The caller
72 # is also expected to abort on the first error so that debugging
73 # can be done by looking at the end of the output.
75 # Each function is documented, implemented and tested independently.
76 # When modifying a helper, the test and the documentation are
77 # expected to be updated and it is easier of they are collocated. A
78 # test for a given function can be run with
80 # ~~~~~~~~~~~~~~~~{.sh}
81 # ceph-helpers.sh TESTS test_get_osds
84 # and all the tests (i.e. all functions matching test_*) are run
87 # ~~~~~~~~~~~~~~~~{.sh}
88 # ceph-helpers.sh TESTS
91 # A test function takes a single argument : the directory dedicated
92 # to the tests. It is expected to not create any file outside of this
93 # directory and remove it entirely when it completes successfully.
98 # Cleanup any leftovers found in **dir** via **teardown**
99 # and reset **dir** as an empty environment.
101 # @param dir path name of the environment
102 # @return 0 on success, 1 on error
106 teardown
$dir ||
return 1
110 function test_setup
() {
112 setup
$dir ||
return 1
113 test -d $dir ||
return 1
114 setup
$dir ||
return 1
115 test -d $dir ||
return 1
119 #######################################################################
122 # Kill all daemons for which a .pid file exists in **dir** and remove
123 # **dir**. If the file system in which **dir** is btrfs, delete all
124 # subvolumes that relate to it.
126 # @param dir path name of the environment
127 # @return 0 on success, 1 on error
129 function teardown
() {
131 kill_daemons
$dir KILL
132 if [ `uname` != FreeBSD
] \
133 && [ $
(stat
-f -c '%T' .
) == "btrfs" ]; then
134 __teardown_btrfs
$dir
139 function __teardown_btrfs
() {
140 local btrfs_base_dir
=$1
141 local btrfs_root
=$
(df
-P . |
tail -1 |
awk '{print $NF}')
142 local btrfs_dirs
=$
(cd $btrfs_base_dir; sudo btrfs subvolume list .
-t |
awk '/^[0-9]/ {print $4}' |
grep "$btrfs_base_dir/$btrfs_dir")
143 for subvolume
in $btrfs_dirs; do
144 sudo btrfs subvolume delete
$btrfs_root/$subvolume
148 function test_teardown
() {
150 setup
$dir ||
return 1
151 teardown
$dir ||
return 1
152 ! test -d $dir ||
return 1
155 #######################################################################
158 # Sends a signal to a single daemon.
159 # This is a helper function for kill_daemons
161 # After the daemon is sent **signal**, its actual termination
162 # will be verified by sending it signal 0. If the daemon is
163 # still alive, kill_daemon will pause for a few seconds and
164 # try again. This will repeat for a fixed number of times
165 # before kill_daemon returns on failure. The list of
166 # sleep intervals can be specified as **delays** and defaults
169 # 0.1 0.2 1 1 1 2 3 5 5 5 10 10 20 60 60 60 120
171 # This sequence is designed to run first a very short sleep time (0.1)
172 # if the machine is fast enough and the daemon terminates in a fraction of a
173 # second. The increasing sleep numbers should give plenty of time for
174 # the daemon to die even on the slowest running machine. If a daemon
175 # takes more than a few minutes to stop (the sum of all sleep times),
176 # there probably is no point in waiting more and a number of things
177 # are likely to go wrong anyway: better give up and return on error.
179 # @param pid the process id to send a signal
180 # @param send_signal the signal to send
181 # @param delays sequence of sleep times before failure
183 function kill_daemon
() {
186 local delays
=${3:-0.1 0.2 1 1 1 2 3 5 5 5 10 10 20 60 60 60 120}
188 for try
in $delays ; do
189 if kill -$send_signal $pid 2> /dev
/null
; then
201 function test_kill_daemon
() {
203 setup
$dir ||
return 1
204 run_mon
$dir a
--osd_pool_default_size=1 ||
return 1
205 run_mgr
$dir x ||
return 1
206 run_osd
$dir 0 ||
return 1
209 for pidfile
in $
(find $dir 2>/dev
/null |
grep $name_prefix'[^/]*\.pid') ; do
211 # sending signal 0 won't kill the daemon
212 # waiting just for one second instead of the default schedule
213 # allows us to quickly verify what happens when kill fails
214 # to stop the daemon (i.e. it must return false)
216 ! kill_daemon
$pidfile 0 1 ||
return 1
218 # killing just the osd and verify the mon still is responsive
220 kill_daemon
$pidfile TERM ||
return 1
223 ceph osd dump |
grep "osd.0 down" ||
return 1
226 for pidfile
in $
(find $dir 2>/dev
/null |
grep $name_prefix'[^/]*\.pid') ; do
230 kill_daemon
$pidfile TERM ||
return 1
234 for pidfile
in $
(find $dir 2>/dev
/null |
grep $name_prefix'[^/]*\.pid') ; do
236 # kill the mon and verify it cannot be reached
238 kill_daemon
$pidfile TERM ||
return 1
239 ! timeout
5 ceph status ||
return 1
242 teardown
$dir ||
return 1
246 # Kill all daemons for which a .pid file exists in **dir**. Each
247 # daemon is sent a **signal** and kill_daemons waits for it to exit
248 # during a few minutes. By default all daemons are killed. If a
249 # **name_prefix** is provided, only the daemons for which a pid
250 # file is found matching the prefix are killed. See run_osd and
251 # run_mon for more information about the name conventions for
254 # Send TERM to all daemons : kill_daemons $dir
255 # Send KILL to all daemons : kill_daemons $dir KILL
256 # Send KILL to all osds : kill_daemons $dir KILL osd
257 # Send KILL to osd 1 : kill_daemons $dir KILL osd.1
259 # If a daemon is sent the TERM signal and does not terminate
260 # within a few minutes, it will still be running even after
261 # kill_daemons returns.
263 # If all daemons are kill successfully the function returns 0
264 # if at least one daemon remains, this is treated as an
265 # error and the function return 1.
267 # @param dir path name of the environment
268 # @param signal name of the first signal (defaults to TERM)
269 # @param name_prefix only kill match daemons (defaults to all)
270 # @param delays sequence of sleep times before failure
271 # @return 0 on success, 1 on error
273 function kill_daemons
() {
274 local trace
=$
(shopt -q -o xtrace
&& echo true ||
echo false
)
275 $trace && shopt -u -o xtrace
277 local signal
=${2:-TERM}
278 local name_prefix
=$3 # optional, osd, mon, osd.1
279 local delays
=$4 #optional timing
283 for pidfile
in $
(find $dir 2>/dev
/null |
grep $name_prefix'[^/]*\.pid') ; do
284 run_in_background pids kill_daemon
$pidfile $signal $delays
290 $trace && shopt -s -o xtrace
294 function test_kill_daemons
() {
296 setup
$dir ||
return 1
297 run_mon
$dir a
--osd_pool_default_size=1 ||
return 1
298 run_mgr
$dir x ||
return 1
299 run_osd
$dir 0 ||
return 1
301 # sending signal 0 won't kill the daemon
302 # waiting just for one second instead of the default schedule
303 # allows us to quickly verify what happens when kill fails
304 # to stop the daemon (i.e. it must return false)
306 ! kill_daemons
$dir 0 osd
1 ||
return 1
308 # killing just the osd and verify the mon still is responsive
310 kill_daemons
$dir TERM osd ||
return 1
311 ceph osd dump |
grep "osd.0 down" ||
return 1
315 kill_daemons
$dir TERM mgr ||
return 1
317 # kill the mon and verify it cannot be reached
319 kill_daemons
$dir TERM ||
return 1
320 ! timeout
5 ceph status ||
return 1
321 teardown
$dir ||
return 1
324 #######################################################################
327 # Run a monitor by the name mon.**id** with data in **dir**/**id**.
328 # The logs can be found in **dir**/mon.**id**.log and the pid file
329 # is **dir**/mon.**id**.pid and the admin socket is
330 # **dir**/**id**/ceph-mon.**id**.asok.
332 # The remaining arguments are passed verbatim to ceph-mon --mkfs
333 # and the ceph-mon daemon.
335 # Two mandatory arguments must be provided: --fsid and --mon-host
336 # Instead of adding them to every call to run_mon, they can be
337 # set in the CEPH_ARGS environment variable to be read implicitly
338 # by every ceph command.
340 # The CEPH_CONF variable is expected to be set to /dev/null to
341 # only rely on arguments for configuration.
345 # CEPH_ARGS="--fsid=$(uuidgen) "
346 # CEPH_ARGS+="--mon-host=127.0.0.1:7018 "
347 # run_mon $dir a # spawn a mon and bind port 7018
348 # run_mon $dir a --debug-filestore=20 # spawn with filestore debugging
350 # If mon_initial_members is not set, the default rbd pool is deleted
351 # and replaced with a replicated pool with less placement groups to
352 # speed up initialization. If mon_initial_members is set, no attempt
353 # is made to recreate the rbd pool because it would hang forever,
354 # waiting for other mons to join.
356 # A **dir**/ceph.conf file is created but not meant to be used by any
357 # function. It is convenient for debugging a failure with:
359 # ceph --conf **dir**/ceph.conf -s
361 # @param dir path name of the environment
362 # @param id mon identifier
363 # @param ... can be any option valid for ceph-mon
364 # @return 0 on success, 1 on error
366 function run_mon_no_pool
() {
382 --mon-osd-full-ratio=.99 \
383 --mon-data-avail-crit=1 \
384 --paxos-propose-interval=0.1 \
385 --osd-crush-chooseleaf-type=0 \
386 --erasure-code-dir=$CEPH_LIB \
387 --plugin-dir=$CEPH_LIB \
393 --log-file=$dir/\
$name.log \
394 --admin-socket=$dir/\
$cluster-\
$name.asok \
395 --mon-cluster-log-file=$dir/log \
397 --pid-file=$dir/\
$name.pid \
398 --mon-allow-pool-delete \
401 cat > $dir/ceph.conf
<<EOF
403 fsid = $(get_config mon $id fsid)
404 mon host = $(get_config mon $id mon_host)
414 run_mon_no_pool
$dir $id "$@" ||
return 1
416 ceph osd pool create rbd
8
418 if test -z "$(get_config mon $id mon_initial_members)" ; then
419 ceph osd pool delete rbd rbd
--yes-i-really-really-mean-it ||
return 1
420 ceph osd pool create rbd
$PG_NUM ||
return 1
421 ceph osd set-backfillfull-ratio
.99
425 function test_run_mon
() {
428 setup
$dir ||
return 1
430 run_mon
$dir a
--mon-initial-members=a ||
return 1
431 # rbd has not been deleted / created, hence it has pool id 0
432 ceph osd dump |
grep "pool 1 'rbd'" ||
return 1
433 kill_daemons
$dir ||
return 1
435 run_mon
$dir a ||
return 1
436 # rbd has been deleted / created, hence it does not have pool id 0
437 ! ceph osd dump |
grep "pool 1 'rbd'" ||
return 1
438 local size
=$
(CEPH_ARGS
='' ceph
--format=json daemon
$dir/ceph-mon.a.asok \
439 config get osd_pool_default_size
)
440 test "$size" = '{"osd_pool_default_size":"3"}' ||
return 1
442 ! CEPH_ARGS
='' ceph status ||
return 1
443 CEPH_ARGS
='' ceph
--conf $dir/ceph.conf status ||
return 1
445 kill_daemons
$dir ||
return 1
447 run_mon
$dir a
--osd_pool_default_size=1 ||
return 1
448 local size
=$
(CEPH_ARGS
='' ceph
--format=json daemon
$dir/ceph-mon.a.asok \
449 config get osd_pool_default_size
)
450 test "$size" = '{"osd_pool_default_size":"1"}' ||
return 1
451 kill_daemons
$dir ||
return 1
453 CEPH_ARGS
="$CEPH_ARGS --osd_pool_default_size=2" \
454 run_mon
$dir a ||
return 1
455 local size
=$
(CEPH_ARGS
='' ceph
--format=json daemon
$dir/ceph-mon.a.asok \
456 config get osd_pool_default_size
)
457 test "$size" = '{"osd_pool_default_size":"2"}' ||
return 1
458 kill_daemons
$dir ||
return 1
460 teardown
$dir ||
return 1
463 #######################################################################
474 --erasure-code-dir=$CEPH_LIB \
475 --plugin-dir=$CEPH_LIB \
477 --debug-objecter 20 \
482 --log-file=$dir/\
$name.log \
483 --admin-socket=$dir/\
$cluster-\
$name.asok \
485 --pid-file=$dir/\
$name.pid \
489 #######################################################################
492 # Create (prepare) and run (activate) an osd by the name osd.**id**
493 # with data in **dir**/**id**. The logs can be found in
494 # **dir**/osd.**id**.log, the pid file is **dir**/osd.**id**.pid and
495 # the admin socket is **dir**/**id**/ceph-osd.**id**.asok.
497 # The remaining arguments are passed verbatim to ceph-osd.
499 # Two mandatory arguments must be provided: --fsid and --mon-host
500 # Instead of adding them to every call to run_osd, they can be
501 # set in the CEPH_ARGS environment variable to be read implicitly
502 # by every ceph command.
504 # The CEPH_CONF variable is expected to be set to /dev/null to
505 # only rely on arguments for configuration.
507 # The run_osd function creates the OSD data directory with ceph-disk
508 # prepare on the **dir**/**id** directory and relies on the
509 # activate_osd function to run the daemon.
513 # CEPH_ARGS="--fsid=$(uuidgen) "
514 # CEPH_ARGS+="--mon-host=127.0.0.1:7018 "
515 # run_osd $dir 0 # prepare and activate an osd using the monitor listening on 7018
517 # @param dir path name of the environment
518 # @param id osd identifier
519 # @param ... can be any option valid for ceph-osd
520 # @return 0 on success, 1 on error
527 local osd_data
=$dir/$id
530 ceph_disk_args
+=" --statedir=$dir"
531 ceph_disk_args
+=" --sysconfdir=$dir"
532 ceph_disk_args
+=" --prepend-to-path="
535 ceph-disk
$ceph_disk_args \
536 prepare
--filestore $osd_data ||
return 1
538 activate_osd
$dir $id "$@"
541 function run_osd_bluestore
() {
546 local osd_data
=$dir/$id
549 ceph_disk_args
+=" --statedir=$dir"
550 ceph_disk_args
+=" --sysconfdir=$dir"
551 ceph_disk_args
+=" --prepend-to-path="
554 ceph-disk
$ceph_disk_args \
555 prepare
--bluestore $osd_data ||
return 1
557 activate_osd
$dir $id "$@"
560 function test_run_osd
() {
563 setup
$dir ||
return 1
565 run_mon
$dir a ||
return 1
566 run_mgr
$dir x ||
return 1
568 run_osd
$dir 0 ||
return 1
569 local backfills
=$
(CEPH_ARGS
='' ceph
--format=json daemon
$dir//ceph-osd
.0.asok \
570 config get osd_max_backfills
)
571 echo "$backfills" |
grep --quiet 'osd_max_backfills' ||
return 1
573 run_osd
$dir 1 --osd-max-backfills 20 ||
return 1
574 local backfills
=$
(CEPH_ARGS
='' ceph
--format=json daemon
$dir//ceph-osd
.1.asok \
575 config get osd_max_backfills
)
576 test "$backfills" = '{"osd_max_backfills":"20"}' ||
return 1
578 CEPH_ARGS
="$CEPH_ARGS --osd-max-backfills 30" run_osd
$dir 2 ||
return 1
579 local backfills
=$
(CEPH_ARGS
='' ceph
--format=json daemon
$dir//ceph-osd
.2.asok \
580 config get osd_max_backfills
)
581 test "$backfills" = '{"osd_max_backfills":"30"}' ||
return 1
583 teardown
$dir ||
return 1
586 #######################################################################
589 # Shutdown and remove all traces of the osd by the name osd.**id**.
591 # The OSD is shutdown with the TERM signal. It is then removed from
592 # the auth list, crush map, osd map etc and the files associated with
593 # it are also removed.
595 # @param dir path name of the environment
596 # @param id osd identifier
597 # @return 0 on success, 1 on error
599 function destroy_osd
() {
603 kill_daemons
$dir TERM osd.
$id ||
return 1
604 ceph osd out osd.
$id ||
return 1
605 ceph auth del osd.
$id ||
return 1
606 ceph osd crush remove osd.
$id ||
return 1
607 ceph osd
rm $id ||
return 1
608 teardown
$dir/$id ||
return 1
612 function test_destroy_osd
() {
615 setup
$dir ||
return 1
616 run_mon
$dir a ||
return 1
617 run_mgr
$dir x ||
return 1
618 run_osd
$dir 0 ||
return 1
619 destroy_osd
$dir 0 ||
return 1
620 ! ceph osd dump |
grep "osd.$id " ||
return 1
621 teardown
$dir ||
return 1
624 #######################################################################
627 # Run (activate) an osd by the name osd.**id** with data in
628 # **dir**/**id**. The logs can be found in **dir**/osd.**id**.log,
629 # the pid file is **dir**/osd.**id**.pid and the admin socket is
630 # **dir**/**id**/ceph-osd.**id**.asok.
632 # The remaining arguments are passed verbatim to ceph-osd.
634 # Two mandatory arguments must be provided: --fsid and --mon-host
635 # Instead of adding them to every call to activate_osd, they can be
636 # set in the CEPH_ARGS environment variable to be read implicitly
637 # by every ceph command.
639 # The CEPH_CONF variable is expected to be set to /dev/null to
640 # only rely on arguments for configuration.
642 # The activate_osd function expects a valid OSD data directory
643 # in **dir**/**id**, either just created via run_osd or re-using
644 # one left by a previous run of ceph-osd. The ceph-osd daemon is
645 # run indirectly via ceph-disk activate.
647 # The activate_osd function blocks until the monitor reports the osd
648 # up. If it fails to do so within $TIMEOUT seconds, activate_osd
653 # CEPH_ARGS="--fsid=$(uuidgen) "
654 # CEPH_ARGS+="--mon-host=127.0.0.1:7018 "
655 # activate_osd $dir 0 # activate an osd using the monitor listening on 7018
657 # @param dir path name of the environment
658 # @param id osd identifier
659 # @param ... can be any option valid for ceph-osd
660 # @return 0 on success, 1 on error
662 function activate_osd
() {
667 local osd_data
=$dir/$id
670 ceph_disk_args
+=" --statedir=$dir"
671 ceph_disk_args
+=" --sysconfdir=$dir"
672 ceph_disk_args
+=" --prepend-to-path="
674 local ceph_args
="$CEPH_ARGS"
675 ceph_args
+=" --osd-failsafe-full-ratio=.99"
676 ceph_args
+=" --osd-journal-size=100"
677 ceph_args
+=" --osd-scrub-load-threshold=2000"
678 ceph_args
+=" --osd-data=$osd_data"
679 ceph_args
+=" --chdir="
680 ceph_args
+=" --erasure-code-dir=$CEPH_LIB"
681 ceph_args
+=" --plugin-dir=$CEPH_LIB"
682 ceph_args
+=" --osd-class-dir=$CEPH_LIB"
683 ceph_args
+=" --run-dir=$dir"
684 ceph_args
+=" --debug-osd=20"
685 ceph_args
+=" --log-file=$dir/\$name.log"
686 ceph_args
+=" --pid-file=$dir/\$name.pid"
687 ceph_args
+=" --osd-max-object-name-len 460"
688 ceph_args
+=" --osd-max-object-namespace-len 64"
689 ceph_args
+=" --enable-experimental-unrecoverable-data-corrupting-features *"
693 CEPH_ARGS
="$ceph_args " ceph-disk
$ceph_disk_args \
696 $osd_data ||
return 1
698 [ "$id" = "$(cat $osd_data/whoami)" ] ||
return 1
700 wait_for_osd up
$id ||
return 1
703 function test_activate_osd
() {
706 setup
$dir ||
return 1
708 run_mon
$dir a ||
return 1
709 run_mgr
$dir x ||
return 1
711 run_osd
$dir 0 ||
return 1
712 local backfills
=$
(CEPH_ARGS
='' ceph
--format=json daemon
$dir//ceph-osd
.0.asok \
713 config get osd_max_backfills
)
714 echo "$backfills" |
grep --quiet 'osd_max_backfills' ||
return 1
716 kill_daemons
$dir TERM osd ||
return 1
718 activate_osd
$dir 0 --osd-max-backfills 20 ||
return 1
719 local backfills
=$
(CEPH_ARGS
='' ceph
--format=json daemon
$dir//ceph-osd
.0.asok \
720 config get osd_max_backfills
)
721 test "$backfills" = '{"osd_max_backfills":"20"}' ||
return 1
723 teardown
$dir ||
return 1
726 #######################################################################
729 # Wait until the OSD **id** is either up or down, as specified by
730 # **state**. It fails after $TIMEOUT seconds.
732 # @param state either up or down
733 # @param id osd identifier
734 # @return 0 on success, 1 on error
736 function wait_for_osd
() {
741 for ((i
=0; i
< $TIMEOUT; i
++)); do
743 if ! ceph osd dump |
grep "osd.$id $state"; then
753 function test_wait_for_osd
() {
755 setup
$dir ||
return 1
756 run_mon
$dir a
--osd_pool_default_size=1 ||
return 1
757 run_mgr
$dir x ||
return 1
758 run_osd
$dir 0 ||
return 1
759 wait_for_osd up
0 ||
return 1
760 kill_daemons
$dir TERM osd ||
return 1
761 wait_for_osd down
0 ||
return 1
762 ( TIMEOUT
=1 ; ! wait_for_osd up
0 ) ||
return 1
763 teardown
$dir ||
return 1
766 #######################################################################
769 # Display the list of OSD ids supporting the **objectname** stored in
770 # **poolname**, as reported by ceph osd map.
772 # @param poolname an existing pool
773 # @param objectname an objectname (may or may not exist)
774 # @param STDOUT white space separated list of OSD ids
775 # @return 0 on success, 1 on error
777 function get_osds
() {
781 local osds
=$
(ceph
--format json osd map
$poolname $objectname 2>/dev
/null | \
783 # get rid of the trailing space
787 function test_get_osds
() {
790 setup
$dir ||
return 1
791 run_mon
$dir a
--osd_pool_default_size=2 ||
return 1
792 run_mgr
$dir x ||
return 1
793 run_osd
$dir 0 ||
return 1
794 run_osd
$dir 1 ||
return 1
795 wait_for_clean ||
return 1
796 get_osds rbd GROUP |
grep --quiet '^[0-1] [0-1]$' ||
return 1
797 teardown
$dir ||
return 1
800 #######################################################################
803 # Wait for the monitor to form quorum (optionally, of size N)
805 # @param timeout duration (lower-bound) to wait for quorum to be formed
806 # @param quorumsize size of quorum to wait for
807 # @return 0 on success, 1 on error
809 function wait_for_quorum
() {
813 if [[ -z "$timeout" ]]; then
817 if [[ -z "$quorumsize" ]]; then
818 timeout
$timeout ceph mon_status
--format=json
>&/dev
/null ||
return 1
823 wait_until
=$
((`date +%s` + $timeout))
824 while [[ $
(date +%s
) -lt $wait_until ]]; do
825 jqfilter
='.quorum | length == '$quorumsize
826 jqinput
="$(timeout $timeout ceph mon_status --format=json 2>/dev/null)"
827 res
=$
(echo $jqinput | jq
"$jqfilter")
828 if [[ "$res" == "true" ]]; then
836 #######################################################################
839 # Return the PG of supporting the **objectname** stored in
840 # **poolname**, as reported by ceph osd map.
842 # @param poolname an existing pool
843 # @param objectname an objectname (may or may not exist)
845 # @return 0 on success, 1 on error
851 ceph
--format json osd map
$poolname $objectname 2>/dev
/null | jq
-r '.pgid'
854 function test_get_pg
() {
857 setup
$dir ||
return 1
858 run_mon
$dir a
--osd_pool_default_size=1 ||
return 1
859 run_mgr
$dir x ||
return 1
860 run_osd
$dir 0 ||
return 1
861 wait_for_clean ||
return 1
862 get_pg rbd GROUP |
grep --quiet '^[0-9]\.[0-9a-f][0-9a-f]*$' ||
return 1
863 teardown
$dir ||
return 1
866 #######################################################################
869 # Return the value of the **config**, obtained via the config get command
870 # of the admin socket of **daemon**.**id**.
872 # @param daemon mon or osd
873 # @param id mon or osd ID
874 # @param config the configuration variable name as found in config_opts.h
875 # @param STDOUT the config value
876 # @return 0 on success, 1 on error
878 function get_config
() {
884 ceph
--format json daemon
$dir/ceph-
$daemon.
$id.asok \
885 config get
$config 2> /dev
/null | \
889 function test_get_config
() {
892 # override the default config using command line arg and check it
893 setup
$dir ||
return 1
894 run_mon
$dir a
--osd_pool_default_size=1 ||
return 1
895 test $
(get_config mon a osd_pool_default_size
) = 1 ||
return 1
896 run_mgr
$dir x ||
return 1
897 run_osd
$dir 0 --osd_max_scrubs=3 ||
return 1
898 test $
(get_config osd
0 osd_max_scrubs
) = 3 ||
return 1
899 teardown
$dir ||
return 1
902 #######################################################################
905 # Set the **config** to specified **value**, via the config set command
906 # of the admin socket of **daemon**.**id**
908 # @param daemon mon or osd
909 # @param id mon or osd ID
910 # @param config the configuration variable name as found in config_opts.h
911 # @param value the config value
912 # @return 0 on success, 1 on error
914 function set_config
() {
920 test $
(env CEPH_ARGS
='' ceph
--format json daemon
$dir/ceph-
$daemon.
$id.asok \
921 config
set $config $value 2> /dev
/null | \
922 jq
'has("success")') == true
925 function test_set_config
() {
928 setup
$dir ||
return 1
929 run_mon
$dir a
--osd_pool_default_size=1 ||
return 1
930 test $
(get_config mon a ms_crc_header
) = true ||
return 1
931 set_config mon a ms_crc_header false ||
return 1
932 test $
(get_config mon a ms_crc_header
) = false ||
return 1
933 set_config mon a ms_crc_header true ||
return 1
934 test $
(get_config mon a ms_crc_header
) = true ||
return 1
935 teardown
$dir ||
return 1
938 #######################################################################
941 # Return the OSD id of the primary OSD supporting the **objectname**
942 # stored in **poolname**, as reported by ceph osd map.
944 # @param poolname an existing pool
945 # @param objectname an objectname (may or may not exist)
946 # @param STDOUT the primary OSD id
947 # @return 0 on success, 1 on error
949 function get_primary
() {
953 ceph
--format json osd map
$poolname $objectname 2>/dev
/null | \
957 function test_get_primary
() {
960 setup
$dir ||
return 1
961 run_mon
$dir a
--osd_pool_default_size=1 ||
return 1
963 run_mgr
$dir x ||
return 1
964 run_osd
$dir $osd ||
return 1
965 wait_for_clean ||
return 1
966 test $
(get_primary rbd GROUP
) = $osd ||
return 1
967 teardown
$dir ||
return 1
970 #######################################################################
973 # Return the id of any OSD supporting the **objectname** stored in
974 # **poolname**, as reported by ceph osd map, except the primary.
976 # @param poolname an existing pool
977 # @param objectname an objectname (may or may not exist)
978 # @param STDOUT the OSD id
979 # @return 0 on success, 1 on error
981 function get_not_primary
() {
985 local primary
=$
(get_primary
$poolname $objectname)
986 ceph
--format json osd map
$poolname $objectname 2>/dev
/null | \
987 jq
".acting | map(select (. != $primary)) | .[0]"
990 function test_get_not_primary
() {
993 setup
$dir ||
return 1
994 run_mon
$dir a
--osd_pool_default_size=2 ||
return 1
995 run_mgr
$dir x ||
return 1
996 run_osd
$dir 0 ||
return 1
997 run_osd
$dir 1 ||
return 1
998 wait_for_clean ||
return 1
999 local primary
=$
(get_primary rbd GROUP
)
1000 local not_primary
=$
(get_not_primary rbd GROUP
)
1001 test $not_primary != $primary ||
return 1
1002 test $not_primary = 0 -o $not_primary = 1 ||
return 1
1003 teardown
$dir ||
return 1
1006 #######################################################################
1009 # Run ceph-objectstore-tool against the OSD **id** using the data path
1010 # **dir**. The OSD is killed with TERM prior to running
1011 # ceph-objectstore-tool because access to the data path is
1012 # exclusive. The OSD is restarted after the command completes. The
1013 # objectstore_tool returns after all PG are active+clean again.
1015 # @param dir the data path of the OSD
1016 # @param id the OSD id
1017 # @param ... arguments to ceph-objectstore-tool
1018 # @param STDIN the input of ceph-objectstore-tool
1019 # @param STDOUT the output of ceph-objectstore-tool
1020 # @return 0 on success, 1 on error
1022 # The value of $ceph_osd_args will be passed to restarted osds
1024 function objectstore_tool
() {
1029 local osd_data
=$dir/$id
1031 local osd_type
=$
(cat $osd_data/type)
1033 kill_daemons
$dir TERM osd.
$id >&2 < /dev
/null ||
return 1
1036 if [ "$objectstore_type" == "filestore" ]; then
1037 journal_args
=" --journal-path $osd_data/journal"
1039 ceph-objectstore-tool \
1040 --data-path $osd_data \
1043 activate_osd
$dir $id $ceph_osd_args >&2 ||
return 1
1047 function test_objectstore_tool
() {
1050 setup
$dir ||
return 1
1051 run_mon
$dir a
--osd_pool_default_size=1 ||
return 1
1053 run_mgr
$dir x ||
return 1
1054 run_osd
$dir $osd ||
return 1
1055 wait_for_clean ||
return 1
1056 rados
--pool rbd put GROUP
/etc
/group ||
return 1
1057 objectstore_tool
$dir $osd GROUP get-bytes | \
1059 ! objectstore_tool
$dir $osd NOTEXISTS get-bytes ||
return 1
1060 teardown
$dir ||
return 1
1063 #######################################################################
1066 # Predicate checking if there is an ongoing recovery in the
1067 # cluster. If any of the recovering_{keys,bytes,objects}_per_sec
1068 # counters are reported by ceph status, it means recovery is in
1071 # @return 0 if recovery in progress, 1 otherwise
1073 function get_is_making_recovery_progress
() {
1074 local recovery_progress
1075 recovery_progress
+=".recovering_keys_per_sec + "
1076 recovery_progress
+=".recovering_bytes_per_sec + "
1077 recovery_progress
+=".recovering_objects_per_sec"
1078 local progress
=$
(ceph
--format json status
2>/dev
/null | \
1079 jq
-r ".pgmap | $recovery_progress")
1080 test "$progress" != null
1083 function test_get_is_making_recovery_progress
() {
1086 setup
$dir ||
return 1
1087 run_mon
$dir a ||
return 1
1088 run_mgr
$dir x ||
return 1
1089 ! get_is_making_recovery_progress ||
return 1
1090 teardown
$dir ||
return 1
1093 #######################################################################
1096 # Return the number of active PGs in the cluster. A PG is active if
1097 # ceph pg dump pgs reports it both **active** and **clean** and that
1100 # @param STDOUT the number of active PGs
1101 # @return 0 on success, 1 on error
1103 function get_num_active_clean
() {
1105 expression
+="select(contains(\"active\") and contains(\"clean\")) | "
1106 expression
+="select(contains(\"stale\") | not)"
1107 ceph
--format json pg dump pgs
2>/dev
/null | \
1108 jq
"[.[] | .state | $expression] | length"
1111 function test_get_num_active_clean
() {
1114 setup
$dir ||
return 1
1115 run_mon
$dir a
--osd_pool_default_size=1 ||
return 1
1116 run_mgr
$dir x ||
return 1
1117 run_osd
$dir 0 ||
return 1
1118 wait_for_clean ||
return 1
1119 local num_active_clean
=$
(get_num_active_clean
)
1120 test "$num_active_clean" = $PG_NUM ||
return 1
1121 teardown
$dir ||
return 1
1124 #######################################################################
1127 # Return the number of PGs in the cluster, according to
1130 # @param STDOUT the number of PGs
1131 # @return 0 on success, 1 on error
1133 function get_num_pgs
() {
1134 ceph
--format json status
2>/dev
/null | jq
'.pgmap.num_pgs'
1137 function test_get_num_pgs
() {
1140 setup
$dir ||
return 1
1141 run_mon
$dir a
--osd_pool_default_size=1 ||
return 1
1142 run_mgr
$dir x ||
return 1
1143 run_osd
$dir 0 ||
return 1
1144 wait_for_clean ||
return 1
1145 local num_pgs
=$
(get_num_pgs
)
1146 test "$num_pgs" -gt 0 ||
return 1
1147 teardown
$dir ||
return 1
1150 #######################################################################
1153 # Return the date and time of the last completed scrub for **pgid**,
1154 # as reported by ceph pg dump pgs. Note that a repair also sets this
1157 # @param pgid the id of the PG
1158 # @param STDOUT the date and time of the last scrub
1159 # @return 0 on success, 1 on error
1161 function get_last_scrub_stamp
() {
1163 local sname
=${2:-last_scrub_stamp}
1164 ceph
--format json pg dump pgs
2>/dev
/null | \
1165 jq
-r ".[] | select(.pgid==\"$pgid\") | .$sname"
1168 function test_get_last_scrub_stamp
() {
1171 setup
$dir ||
return 1
1172 run_mon
$dir a
--osd_pool_default_size=1 ||
return 1
1173 run_mgr
$dir x ||
return 1
1174 run_osd
$dir 0 ||
return 1
1175 wait_for_clean ||
return 1
1176 stamp
=$
(get_last_scrub_stamp
2.0)
1177 test -n "$stamp" ||
return 1
1178 teardown
$dir ||
return 1
1181 #######################################################################
1184 # Predicate checking if the cluster is clean, i.e. all of its PGs are
1185 # in a clean state (see get_num_active_clean for a definition).
1187 # @return 0 if the cluster is clean, 1 otherwise
1189 function is_clean
() {
1190 num_pgs
=$
(get_num_pgs
)
1191 test $num_pgs != 0 ||
return 1
1192 test $
(get_num_active_clean
) = $num_pgs ||
return 1
1195 function test_is_clean
() {
1198 setup
$dir ||
return 1
1199 run_mon
$dir a
--osd_pool_default_size=1 ||
return 1
1200 run_mgr
$dir x ||
return 1
1201 run_osd
$dir 0 ||
return 1
1202 wait_for_clean ||
return 1
1203 is_clean ||
return 1
1204 teardown
$dir ||
return 1
1207 #######################################################################
1210 # Return a list of numbers that are increasingly larger and whose
1211 # total is **timeout** seconds. It can be used to have short sleep
1212 # delay while waiting for an event on a fast machine. But if running
1213 # very slowly the larger delays avoid stressing the machine even
1214 # further or spamming the logs.
1216 # @param timeout sum of all delays, in seconds
1217 # @return a list of sleep delays
1219 function get_timeout_delays
() {
1220 local trace
=$
(shopt -q -o xtrace
&& echo true ||
echo false
)
1221 $trace && shopt -u -o xtrace
1223 local first_step
=${2:-1}
1228 while test "$(echo $total + $i \<= $timeout | bc -l)" = "1"; do
1230 total
=$
(echo $total + $i |
bc -l)
1231 i
=$
(echo $i \
* 2 |
bc -l)
1233 if test "$(echo $total \< $timeout | bc -l)" = "1"; then
1234 echo -n $
(echo $timeout - $total |
bc -l)
1236 $trace && shopt -s -o xtrace
1239 function test_get_timeout_delays
() {
1240 test "$(get_timeout_delays 1)" = "1 " ||
return 1
1241 test "$(get_timeout_delays 5)" = "1 2 2" ||
return 1
1242 test "$(get_timeout_delays 6)" = "1 2 3" ||
return 1
1243 test "$(get_timeout_delays 7)" = "1 2 4 " ||
return 1
1244 test "$(get_timeout_delays 8)" = "1 2 4 1" ||
return 1
1245 test "$(get_timeout_delays 1 .1)" = ".1 .2 .4 .3" ||
return 1
1246 test "$(get_timeout_delays 1.5 .1)" = ".1 .2 .4 .8 " ||
return 1
1247 test "$(get_timeout_delays 5 .1)" = ".1 .2 .4 .8 1.6 1.9" ||
return 1
1248 test "$(get_timeout_delays 6 .1)" = ".1 .2 .4 .8 1.6 2.9" ||
return 1
1249 test "$(get_timeout_delays 6.3 .1)" = ".1 .2 .4 .8 1.6 3.2 " ||
return 1
1250 test "$(get_timeout_delays 20 .1)" = ".1 .2 .4 .8 1.6 3.2 6.4 7.3" ||
return 1
1253 #######################################################################
1256 # Wait until the cluster becomes clean or if it does not make progress
1257 # for $TIMEOUT seconds.
1258 # Progress is measured either via the **get_is_making_recovery_progress**
1259 # predicate or if the number of clean PGs changes (as returned by get_num_active_clean)
1261 # @return 0 if the cluster is clean, 1 otherwise
1263 function wait_for_clean
() {
1264 local num_active_clean
=-1
1265 local cur_active_clean
1266 local -a delays
=($
(get_timeout_delays
$TIMEOUT .1))
1269 while test $
(get_num_pgs
) == 0 ; do
1274 # Comparing get_num_active_clean & get_num_pgs is used to determine
1275 # if the cluster is clean. That's almost an inline of is_clean() to
1276 # get more performance by avoiding multiple calls of get_num_active_clean.
1277 cur_active_clean
=$
(get_num_active_clean
)
1278 test $cur_active_clean = $
(get_num_pgs
) && break
1279 if test $cur_active_clean != $num_active_clean ; then
1281 num_active_clean
=$cur_active_clean
1282 elif get_is_making_recovery_progress
; then
1284 elif (( $loop >= ${#delays[*]} )) ; then
1288 sleep ${delays[$loop]}
1294 function test_wait_for_clean
() {
1297 setup
$dir ||
return 1
1298 run_mon
$dir a
--osd_pool_default_size=1 ||
return 1
1299 run_mgr
$dir x ||
return 1
1300 ! TIMEOUT
=1 wait_for_clean ||
return 1
1301 run_osd
$dir 0 ||
return 1
1302 wait_for_clean ||
return 1
1303 teardown
$dir ||
return 1
1306 #######################################################################
1309 # Wait until the cluster becomes HEALTH_OK again or if it does not make progress
1310 # for $TIMEOUT seconds.
1312 # @return 0 if the cluster is HEALTHY, 1 otherwise
1314 function wait_for_health
() {
1316 local -a delays
=($
(get_timeout_delays
$TIMEOUT .1))
1319 while ! ceph health detail |
grep "$grepstr" ; do
1320 if (( $loop >= ${#delays[*]} )) ; then
1324 sleep ${delays[$loop]}
1329 function wait_for_health_ok
() {
1330 wait_for_health
"HEALTH_OK" ||
return 1
1333 function test_wait_for_health_ok
() {
1336 setup
$dir ||
return 1
1337 run_mon
$dir a
--osd_pool_default_size=1 --osd_failsafe_full_ratio=.99 --mon_pg_warn_min_per_osd=0 ||
return 1
1338 run_mgr
$dir x
--mon_pg_warn_min_per_osd=0 ||
return 1
1339 run_osd
$dir 0 ||
return 1
1340 kill_daemons
$dir TERM osd ||
return 1
1341 ! TIMEOUT
=1 wait_for_health_ok ||
return 1
1342 activate_osd
$dir 0 ||
return 1
1343 wait_for_health_ok ||
return 1
1344 teardown
$dir ||
return 1
1348 #######################################################################
1351 # Run repair on **pgid** and wait until it completes. The repair
1352 # function will fail if repair does not complete within $TIMEOUT
1355 # @param pgid the id of the PG
1356 # @return 0 on success, 1 on error
1360 local last_scrub
=$
(get_last_scrub_stamp
$pgid)
1361 ceph pg repair
$pgid
1362 wait_for_scrub
$pgid "$last_scrub"
1365 function test_repair
() {
1368 setup
$dir ||
return 1
1369 run_mon
$dir a
--osd_pool_default_size=1 ||
return 1
1370 run_mgr
$dir x ||
return 1
1371 run_osd
$dir 0 ||
return 1
1372 wait_for_clean ||
return 1
1373 repair
2.0 ||
return 1
1374 kill_daemons
$dir KILL osd ||
return 1
1375 ! TIMEOUT
=1 repair
2.0 ||
return 1
1376 teardown
$dir ||
return 1
1378 #######################################################################
1381 # Run scrub on **pgid** and wait until it completes. The pg_scrub
1382 # function will fail if repair does not complete within $TIMEOUT
1383 # seconds. The pg_scrub is complete whenever the
1384 # **get_last_scrub_stamp** function reports a timestamp different from
1385 # the one stored before starting the scrub.
1387 # @param pgid the id of the PG
1388 # @return 0 on success, 1 on error
1390 function pg_scrub
() {
1392 local last_scrub
=$
(get_last_scrub_stamp
$pgid)
1394 wait_for_scrub
$pgid "$last_scrub"
1397 function pg_deep_scrub
() {
1399 local last_scrub
=$
(get_last_scrub_stamp
$pgid last_deep_scrub_stamp
)
1400 ceph pg deep-scrub
$pgid
1401 wait_for_scrub
$pgid "$last_scrub" last_deep_scrub_stamp
1404 function test_pg_scrub
() {
1407 setup
$dir ||
return 1
1408 run_mon
$dir a
--osd_pool_default_size=1 ||
return 1
1409 run_mgr
$dir x ||
return 1
1410 run_osd
$dir 0 ||
return 1
1411 wait_for_clean ||
return 1
1412 pg_scrub
2.0 ||
return 1
1413 kill_daemons
$dir KILL osd ||
return 1
1414 ! TIMEOUT
=1 pg_scrub
2.0 ||
return 1
1415 teardown
$dir ||
return 1
1418 #######################################################################
1421 # Run the *command* and expect it to fail (i.e. return a non zero status).
1422 # The output (stderr and stdout) is stored in a temporary file in *dir*
1423 # and is expected to contain the string *expected*.
1425 # Return 0 if the command failed and the string was found. Otherwise
1426 # return 1 and cat the full output of the command on stderr for debug.
1428 # @param dir temporary directory to store the output
1429 # @param expected string to look for in the output
1430 # @param command ... the command and its arguments
1431 # @return 0 on success, 1 on error
1434 function expect_failure
() {
1441 if "$@" > $dir/out
2>&1 ; then
1447 if $success ||
! grep --quiet "$expected" $dir/out
; then
1455 function test_expect_failure
() {
1458 setup
$dir ||
return 1
1459 expect_failure
$dir FAIL bash
-c 'echo FAIL ; exit 1' ||
return 1
1460 # the command did not fail
1461 ! expect_failure
$dir FAIL bash
-c 'echo FAIL ; exit 0' > $dir/out ||
return 1
1462 grep --quiet FAIL
$dir/out ||
return 1
1463 # the command failed but the output does not contain the expected string
1464 ! expect_failure
$dir FAIL bash
-c 'echo UNEXPECTED ; exit 1' > $dir/out ||
return 1
1465 ! grep --quiet FAIL
$dir/out ||
return 1
1466 teardown
$dir ||
return 1
1469 #######################################################################
1472 # Given the *last_scrub*, wait for scrub to happen on **pgid**. It
1473 # will fail if scrub does not complete within $TIMEOUT seconds. The
1474 # repair is complete whenever the **get_last_scrub_stamp** function
1475 # reports a timestamp different from the one given in argument.
1477 # @param pgid the id of the PG
1478 # @param last_scrub timestamp of the last scrub for *pgid*
1479 # @return 0 on success, 1 on error
1481 function wait_for_scrub
() {
1483 local last_scrub
="$2"
1484 local sname
=${3:-last_scrub_stamp}
1486 for ((i
=0; i
< $TIMEOUT; i
++)); do
1487 if test "$last_scrub" != "$(get_last_scrub_stamp $pgid $sname)" ; then
1495 function test_wait_for_scrub
() {
1498 setup
$dir ||
return 1
1499 run_mon
$dir a
--osd_pool_default_size=1 ||
return 1
1500 run_mgr
$dir x ||
return 1
1501 run_osd
$dir 0 ||
return 1
1502 wait_for_clean ||
return 1
1504 ceph pg repair
$pgid
1505 local last_scrub
=$
(get_last_scrub_stamp
$pgid)
1506 wait_for_scrub
$pgid "$last_scrub" ||
return 1
1507 kill_daemons
$dir KILL osd ||
return 1
1508 last_scrub
=$
(get_last_scrub_stamp
$pgid)
1509 ! TIMEOUT
=1 wait_for_scrub
$pgid "$last_scrub" ||
return 1
1510 teardown
$dir ||
return 1
1513 #######################################################################
1516 # Return 0 if the erasure code *plugin* is available, 1 otherwise.
1518 # @param plugin erasure code plugin
1519 # @return 0 on success, 1 on error
1522 function erasure_code_plugin_exists
() {
1528 FreeBSD
) grepstr
="Cannot open.*$plugin" ;;
1529 *) grepstr
="$plugin.*No such file" ;;
1532 s
=$
(ceph osd erasure-code-profile
set TESTPROFILE plugin
=$plugin 2>&1)
1534 if [ $status -eq 0 ]; then
1535 ceph osd erasure-code-profile
rm TESTPROFILE
1536 elif ! echo $s |
grep --quiet "$grepstr" ; then
1538 # display why the string was rejected.
1544 function test_erasure_code_plugin_exists
() {
1547 setup
$dir ||
return 1
1548 run_mon
$dir a ||
return 1
1549 run_mgr
$dir x ||
return 1
1550 erasure_code_plugin_exists jerasure ||
return 1
1551 ! erasure_code_plugin_exists FAKE ||
return 1
1552 teardown
$dir ||
return 1
1555 #######################################################################
1558 # Display all log files from **dir** on stdout.
1560 # @param dir directory in which all data is stored
1563 function display_logs
() {
1566 find $dir -maxdepth 1 -name '*.log' | \
1567 while read file ; do
1568 echo "======================= $file"
1573 function test_display_logs
() {
1576 setup
$dir ||
return 1
1577 run_mon
$dir a ||
return 1
1578 kill_daemons
$dir ||
return 1
1579 display_logs
$dir > $dir/log.out
1580 grep --quiet mon.a.log
$dir/log.out ||
return 1
1581 teardown
$dir ||
return 1
1584 #######################################################################
1586 # Spawn a command in background and save the pid in the variable name
1587 # passed in argument. To make the output reading easier, the output is
1588 # prepend with the process id.
1592 # run_in_background pids1 bash -c 'sleep 1; exit 1'
1594 # @param pid_variable the variable name (not value) where the pids will be stored
1595 # @param ... the command to execute
1596 # @return only the pid_variable output should be considered and used with **wait_background**
1598 function run_in_background
() {
1599 local pid_variable
=$1
1601 # Execute the command and prepend the output with its pid
1602 # We enforce to return the exit status of the command and not the awk one.
1603 ("$@" |
& awk '{ a[i++] = $0 }END{for (i = 0; i in a; ++i) { print "'$$
': " a[i]} }'; return ${PIPESTATUS[0]}) >&2 &
1604 eval "$pid_variable+=\" $!\""
1607 function test_run_in_background
() {
1609 run_in_background pids
sleep 1
1610 run_in_background pids
sleep 1
1611 test $
(echo $pids |
wc -w) = 2 ||
return 1
1612 wait $pids ||
return 1
1615 #######################################################################
1617 # Wait for pids running in background to complete.
1618 # This function is usually used after a **run_in_background** call
1621 # run_in_background pids1 bash -c 'sleep 1; exit 1'
1622 # wait_background pids1
1624 # @param pids The variable name that contains the active PIDS. Set as empty at then end of the function.
1625 # @return returns 1 if at least one process exits in error unless returns 0
1627 function wait_background
() {
1628 # We extract the PIDS from the variable name
1632 for pid
in $pids; do
1633 if ! wait $pid; then
1634 # If one process failed then return 1
1639 # We empty the variable reporting that all process ended
1646 function test_wait_background
() {
1648 run_in_background pids bash
-c "sleep 1; exit 1"
1649 run_in_background pids bash
-c "sleep 2; exit 0"
1650 wait_background pids
1651 if [ $?
-ne 1 ]; then return 1; fi
1653 run_in_background pids bash
-c "sleep 1; exit 0"
1654 run_in_background pids bash
-c "sleep 2; exit 0"
1655 wait_background pids
1656 if [ $?
-ne 0 ]; then return 1; fi
1658 if [ ! -z "$pids" ]; then return 1; fi
1661 function flush_pg_stats
()
1663 local timeout
=${1:-$TIMEOUT}
1668 seq=`ceph tell osd.$osd flush_pg_stats`
1669 seqs
="$seqs $osd-$seq"
1673 osd
=`echo $s | cut -d - -f 1`
1674 seq=`echo $s | cut -d - -f 2`
1675 echo "waiting osd.$osd seq $seq"
1676 while test $
(ceph osd last-stat-seq
$osd) -lt $seq; do
1678 if [ $
((timeout--
)) -eq 0 ]; then
1685 function test_flush_pg_stats
()
1689 setup
$dir ||
return 1
1690 run_mon
$dir a
--osd_pool_default_size=1 ||
return 1
1691 run_mgr
$dir x ||
return 1
1692 run_osd
$dir 0 ||
return 1
1693 rados
-p rbd put obj
/etc
/group
1695 local jq_filter
='.pools | .[] | select(.name == "rbd") | .stats'
1696 raw_bytes_used
=`ceph df detail --format=json | jq "$jq_filter.raw_bytes_used"`
1697 bytes_used
=`ceph df detail --format=json | jq "$jq_filter.bytes_used"`
1698 test $raw_bytes_used > 0 ||
return 1
1699 test $raw_bytes_used == $bytes_used ||
return 1
1702 #######################################################################
1705 # Call the **run** function (which must be defined by the caller) with
1706 # the **dir** argument followed by the caller argument list.
1708 # If the **run** function returns on error, all logs found in **dir**
1709 # are displayed for diagnostic purposes.
1711 # **teardown** function is called when the **run** function returns
1712 # (on success or on error), to cleanup leftovers. The CEPH_CONF is set
1713 # to /dev/null and CEPH_ARGS is unset so that the tests are protected from
1714 # external interferences.
1716 # It is the responsibility of the **run** function to call the
1717 # **setup** function to prepare the test environment (create a temporary
1720 # The shell is required (via PS4) to display the function and line
1721 # number whenever a statement is executed to help debugging.
1723 # @param dir directory in which all data is stored
1724 # @param ... arguments passed transparently to **run**
1725 # @return 0 on success, 1 on error
1732 PS4
='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: '
1734 export PATH
=${CEPH_BUILD_VIRTUALENV}/ceph-disk-virtualenv
/bin
:${CEPH_BUILD_VIRTUALENV}/ceph-detect-init-virtualenv
/bin
:.
:$PATH # make sure program from sources are preferred
1735 #export PATH=$CEPH_ROOT/src/ceph-disk/virtualenv/bin:$CEPH_ROOT/src/ceph-detect-init/virtualenv/bin:.:$PATH # make sure program from sources are preferred
1737 export CEPH_CONF
=/dev
/null
1741 if run
$dir "$@" ; then
1747 teardown
$dir ||
return 1
1751 #######################################################################
1753 function run_tests
() {
1755 PS4
='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: '
1757 export PATH
=${CEPH_BUILD_VIRTUALENV}/ceph-disk-virtualenv
/bin
:${CEPH_BUILD_VIRTUALENV}/ceph-detect-init-virtualenv
/bin
:.
:$PATH # make sure program from sources are preferred
1758 #export PATH=$CEPH_ROOT/src/ceph-disk/virtualenv/bin:$CEPH_ROOT/src/ceph-detect-init/virtualenv/bin:.:$PATH # make sure program from sources are preferred
1760 export CEPH_MON
="127.0.0.1:7109" # git grep '\<7109\>' : there must be only one
1762 CEPH_ARGS
+="--fsid=$(uuidgen) --auth-supported=none "
1763 CEPH_ARGS
+="--mon-host=$CEPH_MON "
1764 export CEPH_CONF
=/dev
/null
1766 local funcs
=${@:-$(set | sed -n -e 's/^\(test_[0-9a-z_]*\) .*/\1/p')}
1767 local dir
=td
/ceph-helpers
1769 for func
in $funcs ; do
1770 $func $dir ||
return 1
1774 if test "$1" = TESTS
; then
1780 # jq only support --exit-status|-e from version 1.4 forwards, which makes
1781 # returning on error waaaay prettier and straightforward.
1782 # However, the current automated upstream build is running with v1.3,
1783 # which has no idea what -e is. Hence the convoluted error checking we
1785 # The next time someone changes this code, please check if v1.4 is now
1786 # a thing, and, if so, please change these to use -e. Thanks.
1788 # jq '.all.supported | select([.[] == "foo"] | any)'
1789 function jq_success
() {
1794 in_escaped
=$
(printf %s
"$input" |
sed "s/'/'\\\\''/g")
1795 filter_escaped
=$
(printf %s
"$filter" |
sed "s/'/'\\\\''/g")
1797 ret
=$
(echo "$in_escaped" | jq
"$filter_escaped")
1798 if [[ "$ret" == "true" ]]; then
1800 elif [[ -n "$expects" ]]; then
1801 if [[ "$ret" == "$expects" ]]; then
1810 ret
="$(echo $input | jq \"$filter\")"
1811 if [[ "$ret" == "true" ]]; then
1813 elif [[ -n "$expects" && "$ret" == "$expects" ]]; then
1820 # compile-command: "cd ../../src ; make -j4 && ../qa/workunits/ceph-helpers.sh TESTS # test_get_config"