]>
Commit | Line | Data |
---|---|---|
b32b8144 FG |
1 | #!/usr/bin/env bash |
2 | # | |
3 | # Copyright (C) 2017 Red Hat <contact@redhat.com> | |
4 | # | |
5 | # | |
6 | # Author: Kefu Chai <kchai@redhat.com> | |
7 | # Author: David Zafman <dzafman@redhat.com> | |
8 | # | |
9 | # This program is free software; you can redistribute it and/or modify | |
10 | # it under the terms of the GNU Library Public License as published by | |
11 | # the Free Software Foundation; either version 2, or (at your option) | |
12 | # any later version. | |
13 | # | |
14 | # This program is distributed in the hope that it will be useful, | |
15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
17 | # GNU Library Public License for more details. | |
18 | # | |
19 | ||
20 | source $CEPH_ROOT/qa/standalone/ceph-helpers.sh | |
21 | ||
22 | function run() { | |
23 | local dir=$1 | |
24 | shift | |
25 | ||
26 | export CEPH_MON="127.0.0.1:7140" # git grep '\<7140\>' : there must be only one | |
27 | export CEPH_ARGS | |
28 | CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " | |
29 | CEPH_ARGS+="--mon-host=$CEPH_MON " | |
30 | ||
31 | local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} | |
32 | for func in $funcs ; do | |
33 | setup $dir || return 1 | |
34 | run_mon $dir a || return 1 | |
35 | run_mgr $dir x || return 1 | |
36 | create_rbd_pool || return 1 | |
37 | ||
38 | $func $dir || return 1 | |
39 | teardown $dir || return 1 | |
40 | done | |
41 | } | |
42 | ||
43 | function setup_osds() { | |
44 | local count=$1 | |
45 | shift | |
46 | ||
47 | for id in $(seq 0 $(expr $count - 1)) ; do | |
48 | run_osd $dir $id || return 1 | |
49 | done | |
50 | wait_for_clean || return 1 | |
51 | } | |
52 | ||
53 | function get_state() { | |
54 | local pgid=$1 | |
55 | local sname=state | |
56 | ceph --format json pg dump pgs 2>/dev/null | \ | |
57 | jq -r ".[] | select(.pgid==\"$pgid\") | .$sname" | |
58 | } | |
59 | ||
b32b8144 FG |
60 | function rados_put() { |
61 | local dir=$1 | |
62 | local poolname=$2 | |
63 | local objname=${3:-SOMETHING} | |
64 | ||
65 | for marker in AAA BBB CCCC DDDD ; do | |
66 | printf "%*s" 1024 $marker | |
67 | done > $dir/ORIGINAL | |
68 | # | |
69 | # get and put an object, compare they are equal | |
70 | # | |
71 | rados --pool $poolname put $objname $dir/ORIGINAL || return 1 | |
72 | } | |
73 | ||
74 | function rados_get() { | |
75 | local dir=$1 | |
76 | local poolname=$2 | |
77 | local objname=${3:-SOMETHING} | |
78 | local expect=${4:-ok} | |
79 | ||
80 | # | |
81 | # Expect a failure to get object | |
82 | # | |
83 | if [ $expect = "fail" ]; | |
84 | then | |
85 | ! rados --pool $poolname get $objname $dir/COPY | |
86 | return | |
87 | fi | |
88 | # | |
89 | # Expect hang trying to get object | |
90 | # | |
91 | if [ $expect = "hang" ]; | |
92 | then | |
93 | timeout 5 rados --pool $poolname get $objname $dir/COPY | |
94 | test "$?" = "124" | |
95 | return | |
96 | fi | |
97 | # | |
98 | # get an object, compare with $dir/ORIGINAL | |
99 | # | |
100 | rados --pool $poolname get $objname $dir/COPY || return 1 | |
101 | diff $dir/ORIGINAL $dir/COPY || return 1 | |
102 | rm $dir/COPY | |
103 | } | |
104 | ||
105 | function rados_get_data() { | |
106 | local inject=$1 | |
107 | shift | |
108 | local dir=$1 | |
109 | ||
110 | local poolname=pool-rep | |
111 | local objname=obj-$inject-$$ | |
112 | rados_put $dir $poolname $objname || return 1 | |
113 | inject_$inject rep data $poolname $objname $dir 0 || return 1 | |
114 | rados_get $dir $poolname $objname || return 1 | |
115 | ||
116 | inject_$inject rep data $poolname $objname $dir 0 || return 1 | |
117 | inject_$inject rep data $poolname $objname $dir 1 || return 1 | |
118 | rados_get $dir $poolname $objname || return 1 | |
119 | ||
120 | inject_$inject rep data $poolname $objname $dir 0 || return 1 | |
121 | inject_$inject rep data $poolname $objname $dir 1 || return 1 | |
122 | inject_$inject rep data $poolname $objname $dir 2 || return 1 | |
123 | rados_get $dir $poolname $objname hang || return 1 | |
124 | } | |
125 | ||
126 | function TEST_rados_get_with_eio() { | |
127 | local dir=$1 | |
128 | ||
129 | setup_osds 4 || return 1 | |
130 | ||
131 | local poolname=pool-rep | |
132 | create_pool $poolname 1 1 || return 1 | |
133 | wait_for_clean || return 1 | |
134 | rados_get_data eio $dir || return 1 | |
135 | ||
136 | delete_pool $poolname | |
137 | } | |
138 | ||
139 | # Test backfill with unfound object | |
140 | function TEST_rep_backfill_unfound() { | |
141 | local dir=$1 | |
142 | local objname=myobject | |
143 | local lastobj=300 | |
144 | # Must be between 1 and $lastobj | |
145 | local testobj=obj250 | |
146 | ||
147 | export CEPH_ARGS | |
148 | CEPH_ARGS+=' --osd_min_pg_log_entries=5 --osd_max_pg_log_entries=10' | |
149 | setup_osds 3 || return 1 | |
150 | ||
151 | local poolname=test-pool | |
152 | create_pool $poolname 1 1 || return 1 | |
153 | wait_for_clean || return 1 | |
154 | ||
155 | ceph pg dump pgs | |
156 | ||
157 | rados_put $dir $poolname $objname || return 1 | |
158 | ||
159 | local -a initial_osds=($(get_osds $poolname $objname)) | |
160 | local last_osd=${initial_osds[-1]} | |
161 | kill_daemons $dir TERM osd.${last_osd} 2>&2 < /dev/null || return 1 | |
162 | ceph osd down ${last_osd} || return 1 | |
163 | ceph osd out ${last_osd} || return 1 | |
164 | ||
165 | ceph pg dump pgs | |
166 | ||
167 | dd if=/dev/urandom of=${dir}/ORIGINAL bs=1024 count=4 | |
168 | for i in $(seq 1 $lastobj) | |
169 | do | |
170 | rados --pool $poolname put obj${i} $dir/ORIGINAL || return 1 | |
171 | done | |
172 | ||
173 | inject_eio rep data $poolname $testobj $dir 0 || return 1 | |
174 | inject_eio rep data $poolname $testobj $dir 1 || return 1 | |
175 | ||
176 | run_osd $dir ${last_osd} || return 1 | |
177 | ceph osd in ${last_osd} || return 1 | |
178 | ||
179 | sleep 15 | |
180 | ||
181 | for tmp in $(seq 1 100); do | |
182 | state=$(get_state 2.0) | |
183 | echo $state | grep backfill_unfound | |
184 | if [ "$?" = "0" ]; then | |
185 | break | |
186 | fi | |
187 | echo "$state " | |
188 | sleep 1 | |
189 | done | |
190 | ||
191 | ceph pg dump pgs | |
192 | ceph pg 2.0 list_missing | grep -q $testobj || return 1 | |
193 | ||
194 | # Command should hang because object is unfound | |
195 | timeout 5 rados -p $poolname get $testobj $dir/CHECK | |
196 | test $? = "124" || return 1 | |
197 | ||
198 | ceph pg 2.0 mark_unfound_lost delete | |
199 | ||
200 | wait_for_clean || return 1 | |
201 | ||
202 | for i in $(seq 1 $lastobj) | |
203 | do | |
204 | if [ obj${i} = "$testobj" ]; then | |
205 | # Doesn't exist anymore | |
206 | ! rados -p $poolname get $testobj $dir/CHECK || return 1 | |
207 | else | |
208 | rados --pool $poolname get obj${i} $dir/CHECK || return 1 | |
209 | diff -q $dir/ORIGINAL $dir/CHECK || return 1 | |
210 | fi | |
211 | done | |
212 | ||
213 | rm -f ${dir}/ORIGINAL ${dir}/CHECK | |
214 | ||
215 | delete_pool $poolname | |
216 | } | |
217 | ||
218 | # Test recovery with unfound object | |
219 | function TEST_rep_recovery_unfound() { | |
220 | local dir=$1 | |
221 | local objname=myobject | |
222 | local lastobj=100 | |
223 | # Must be between 1 and $lastobj | |
224 | local testobj=obj75 | |
225 | ||
226 | setup_osds 3 || return 1 | |
227 | ||
228 | local poolname=test-pool | |
229 | create_pool $poolname 1 1 || return 1 | |
230 | wait_for_clean || return 1 | |
231 | ||
232 | ceph pg dump pgs | |
233 | ||
234 | rados_put $dir $poolname $objname || return 1 | |
235 | ||
236 | local -a initial_osds=($(get_osds $poolname $objname)) | |
237 | local last_osd=${initial_osds[-1]} | |
238 | kill_daemons $dir TERM osd.${last_osd} 2>&2 < /dev/null || return 1 | |
239 | ceph osd down ${last_osd} || return 1 | |
240 | ceph osd out ${last_osd} || return 1 | |
241 | ||
242 | ceph pg dump pgs | |
243 | ||
244 | dd if=/dev/urandom of=${dir}/ORIGINAL bs=1024 count=4 | |
245 | for i in $(seq 1 $lastobj) | |
246 | do | |
247 | rados --pool $poolname put obj${i} $dir/ORIGINAL || return 1 | |
248 | done | |
249 | ||
250 | inject_eio rep data $poolname $testobj $dir 0 || return 1 | |
251 | inject_eio rep data $poolname $testobj $dir 1 || return 1 | |
252 | ||
253 | run_osd $dir ${last_osd} || return 1 | |
254 | ceph osd in ${last_osd} || return 1 | |
255 | ||
256 | sleep 15 | |
257 | ||
258 | for tmp in $(seq 1 100); do | |
259 | state=$(get_state 2.0) | |
260 | echo $state | grep -v recovering | |
261 | if [ "$?" = "0" ]; then | |
262 | break | |
263 | fi | |
264 | echo "$state " | |
265 | sleep 1 | |
266 | done | |
267 | ||
268 | ceph pg dump pgs | |
269 | ceph pg 2.0 list_missing | grep -q $testobj || return 1 | |
270 | ||
271 | # Command should hang because object is unfound | |
272 | timeout 5 rados -p $poolname get $testobj $dir/CHECK | |
273 | test $? = "124" || return 1 | |
274 | ||
275 | ceph pg 2.0 mark_unfound_lost delete | |
276 | ||
277 | wait_for_clean || return 1 | |
278 | ||
279 | for i in $(seq 1 $lastobj) | |
280 | do | |
281 | if [ obj${i} = "$testobj" ]; then | |
282 | # Doesn't exist anymore | |
283 | ! rados -p $poolname get $testobj $dir/CHECK || return 1 | |
284 | else | |
285 | rados --pool $poolname get obj${i} $dir/CHECK || return 1 | |
286 | diff -q $dir/ORIGINAL $dir/CHECK || return 1 | |
287 | fi | |
288 | done | |
289 | ||
290 | rm -f ${dir}/ORIGINAL ${dir}/CHECK | |
291 | ||
292 | delete_pool $poolname | |
293 | } | |
294 | ||
295 | main osd-rep-recov-eio.sh "$@" | |
296 | ||
297 | # Local Variables: | |
298 | # compile-command: "cd ../../../build ; make -j4 && ../qa/run-standalone.sh osd-rep-recov-eio.sh" | |
299 | # End: |