]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | #!/bin/bash |
2 | # | |
3 | # Copyright (C) 2016 Piotr Dałek <git@predictor.org.pl> | |
4 | # Copyright (C) 2014, 2015 Red Hat <contact@redhat.com> | |
5 | # | |
6 | # Author: Piotr Dałek <git@predictor.org.pl> | |
7 | # | |
8 | # This program is free software; you can redistribute it and/or modify | |
9 | # it under the terms of the GNU Library Public License as published by | |
10 | # the Free Software Foundation; either version 2, or (at your option) | |
11 | # any later version. | |
12 | # | |
13 | # This program is distributed in the hope that it will be useful, | |
14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
16 | # GNU Library Public License for more details. | |
17 | # | |
18 | ||
c07f9fc5 | 19 | source $CEPH_ROOT/qa/standalone/ceph-helpers.sh |
7c673cae FG |
20 | MAX_PROPAGATION_TIME=30 |
21 | ||
22 | function run() { | |
23 | local dir=$1 | |
24 | shift | |
25 | rm -f $dir/*.pid | |
26 | export CEPH_MON="127.0.0.1:7126" # git grep '\<7126\>' : there must be only one | |
27 | export CEPH_ARGS | |
28 | CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " | |
29 | CEPH_ARGS+="--mon-host=$CEPH_MON " | |
c07f9fc5 | 30 | |
7c673cae FG |
31 | OLD_ARGS=$CEPH_ARGS |
32 | CEPH_ARGS+="--osd-fast-fail-on-connection-refused=false " | |
33 | echo "Ensuring old behavior is there..." | |
34 | test_fast_kill $dir && (echo "OSDs died too early! Old behavior doesn't work." ; return 1) | |
35 | ||
36 | CEPH_ARGS=$OLD_ARGS"--osd-fast-fail-on-connection-refused=true " | |
37 | OLD_ARGS=$CEPH_ARGS | |
38 | ||
39 | CEPH_ARGS+="--ms_type=simple" | |
40 | echo "Testing simple msgr..." | |
41 | test_fast_kill $dir || return 1 | |
42 | ||
43 | CEPH_ARGS=$OLD_ARGS"--ms_type=async" | |
44 | echo "Testing async msgr..." | |
45 | test_fast_kill $dir || return 1 | |
46 | ||
47 | return 0 | |
48 | ||
49 | } | |
50 | ||
51 | function test_fast_kill() { | |
52 | # create cluster with 3 osds | |
53 | setup $dir || return 1 | |
54 | run_mon $dir a --osd_pool_default_size=3 || return 1 | |
224ce89b | 55 | run_mgr $dir x || return 1 |
7c673cae FG |
56 | for oi in {0..2}; do |
57 | run_osd $dir $oi || return 1 | |
58 | pids[$oi]=$(cat $dir/osd.$oi.pid) | |
59 | done | |
60 | ||
c07f9fc5 FG |
61 | create_rbd_pool || return 1 |
62 | ||
7c673cae FG |
63 | # make some objects so osds to ensure connectivity between osds |
64 | rados -p rbd bench 10 write -b 4096 --max-objects 128 --no-cleanup | |
65 | sleep 1 | |
66 | ||
67 | killid=0 | |
68 | previd=0 | |
69 | ||
70 | # kill random osd and see if after max MAX_PROPAGATION_TIME, the osd count decreased. | |
71 | for i in {1..2}; do | |
72 | while [ $killid -eq $previd ]; do | |
73 | killid=${pids[$RANDOM%${#pids[@]}]} | |
74 | done | |
75 | previd=$killid | |
76 | ||
77 | kill -9 $killid | |
78 | time_left=$MAX_PROPAGATION_TIME | |
79 | down_osds=0 | |
c07f9fc5 | 80 | |
7c673cae FG |
81 | while [ $time_left -gt 0 ]; do |
82 | sleep 1 | |
83 | time_left=$[$time_left - 1]; | |
84 | ||
85 | grep -m 1 -c -F "ms_handle_refused" $dir/osd.*.log > /dev/null | |
86 | if [ $? -ne 0 ]; then | |
87 | continue | |
88 | fi | |
89 | ||
90 | down_osds=$(ceph osd tree | grep -c down) | |
91 | if [ $down_osds -lt $i ]; then | |
92 | # osds not marked down yet, try again in a second | |
93 | continue | |
94 | elif [ $down_osds -gt $i ]; then | |
95 | echo Too many \($down_osds\) osds died! | |
96 | return 1 | |
97 | else | |
98 | break | |
99 | fi | |
100 | done | |
101 | ||
102 | if [ $down_osds -lt $i ]; then | |
103 | echo Killed the OSD, yet it is not marked down | |
104 | ceph osd tree | |
105 | return 1 | |
106 | fi | |
107 | done | |
108 | pkill -SIGTERM rados | |
109 | teardown $dir || return 1 | |
110 | } | |
111 | ||
112 | main osd-fast-mark-down "$@" | |
113 | ||
114 | # Local Variables: | |
115 | # compile-command: "cd ../.. ; make -j4 && test/osd/osd-fast-mark-down.sh" | |
116 | # End: |