]>
Commit | Line | Data |
---|---|---|
b32b8144 FG |
1 | #!/usr/bin/env bash |
2 | # | |
3 | # Copyright (C) 2017 Red Hat <contact@redhat.com> | |
4 | # | |
5 | # | |
6 | # Author: Kefu Chai <kchai@redhat.com> | |
7 | # Author: David Zafman <dzafman@redhat.com> | |
8 | # | |
9 | # This program is free software; you can redistribute it and/or modify | |
10 | # it under the terms of the GNU Library Public License as published by | |
11 | # the Free Software Foundation; either version 2, or (at your option) | |
12 | # any later version. | |
13 | # | |
14 | # This program is distributed in the hope that it will be useful, | |
15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
17 | # GNU Library Public License for more details. | |
18 | # | |
19 | ||
20 | source $CEPH_ROOT/qa/standalone/ceph-helpers.sh | |
21 | ||
22 | function run() { | |
23 | local dir=$1 | |
24 | shift | |
25 | ||
26 | export CEPH_MON="127.0.0.1:7140" # git grep '\<7140\>' : there must be only one | |
27 | export CEPH_ARGS | |
28 | CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " | |
29 | CEPH_ARGS+="--mon-host=$CEPH_MON " | |
30 | ||
31 | local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} | |
32 | for func in $funcs ; do | |
33 | setup $dir || return 1 | |
34 | run_mon $dir a || return 1 | |
35 | run_mgr $dir x || return 1 | |
36 | create_rbd_pool || return 1 | |
37 | ||
38 | $func $dir || return 1 | |
39 | teardown $dir || return 1 | |
40 | done | |
41 | } | |
42 | ||
43 | function setup_osds() { | |
44 | local count=$1 | |
45 | shift | |
46 | ||
47 | for id in $(seq 0 $(expr $count - 1)) ; do | |
48 | run_osd $dir $id || return 1 | |
49 | done | |
50 | wait_for_clean || return 1 | |
51 | } | |
52 | ||
53 | function get_state() { | |
54 | local pgid=$1 | |
55 | local sname=state | |
56 | ceph --format json pg dump pgs 2>/dev/null | \ | |
57 | jq -r ".[] | select(.pgid==\"$pgid\") | .$sname" | |
58 | } | |
59 | ||
60 | function delete_pool() { | |
61 | local poolname=$1 | |
62 | ||
63 | ceph osd pool delete $poolname $poolname --yes-i-really-really-mean-it | |
64 | } | |
65 | ||
66 | function rados_put() { | |
67 | local dir=$1 | |
68 | local poolname=$2 | |
69 | local objname=${3:-SOMETHING} | |
70 | ||
71 | for marker in AAA BBB CCCC DDDD ; do | |
72 | printf "%*s" 1024 $marker | |
73 | done > $dir/ORIGINAL | |
74 | # | |
75 | # get and put an object, compare they are equal | |
76 | # | |
77 | rados --pool $poolname put $objname $dir/ORIGINAL || return 1 | |
78 | } | |
79 | ||
80 | function rados_get() { | |
81 | local dir=$1 | |
82 | local poolname=$2 | |
83 | local objname=${3:-SOMETHING} | |
84 | local expect=${4:-ok} | |
85 | ||
86 | # | |
87 | # Expect a failure to get object | |
88 | # | |
89 | if [ $expect = "fail" ]; | |
90 | then | |
91 | ! rados --pool $poolname get $objname $dir/COPY | |
92 | return | |
93 | fi | |
94 | # | |
95 | # Expect hang trying to get object | |
96 | # | |
97 | if [ $expect = "hang" ]; | |
98 | then | |
99 | timeout 5 rados --pool $poolname get $objname $dir/COPY | |
100 | test "$?" = "124" | |
101 | return | |
102 | fi | |
103 | # | |
104 | # get an object, compare with $dir/ORIGINAL | |
105 | # | |
106 | rados --pool $poolname get $objname $dir/COPY || return 1 | |
107 | diff $dir/ORIGINAL $dir/COPY || return 1 | |
108 | rm $dir/COPY | |
109 | } | |
110 | ||
111 | function rados_get_data() { | |
112 | local inject=$1 | |
113 | shift | |
114 | local dir=$1 | |
115 | ||
116 | local poolname=pool-rep | |
117 | local objname=obj-$inject-$$ | |
118 | rados_put $dir $poolname $objname || return 1 | |
119 | inject_$inject rep data $poolname $objname $dir 0 || return 1 | |
120 | rados_get $dir $poolname $objname || return 1 | |
121 | ||
122 | inject_$inject rep data $poolname $objname $dir 0 || return 1 | |
123 | inject_$inject rep data $poolname $objname $dir 1 || return 1 | |
124 | rados_get $dir $poolname $objname || return 1 | |
125 | ||
126 | inject_$inject rep data $poolname $objname $dir 0 || return 1 | |
127 | inject_$inject rep data $poolname $objname $dir 1 || return 1 | |
128 | inject_$inject rep data $poolname $objname $dir 2 || return 1 | |
129 | rados_get $dir $poolname $objname hang || return 1 | |
130 | } | |
131 | ||
132 | function TEST_rados_get_with_eio() { | |
133 | local dir=$1 | |
134 | ||
135 | setup_osds 4 || return 1 | |
136 | ||
137 | local poolname=pool-rep | |
138 | create_pool $poolname 1 1 || return 1 | |
139 | wait_for_clean || return 1 | |
140 | rados_get_data eio $dir || return 1 | |
141 | ||
142 | delete_pool $poolname | |
143 | } | |
144 | ||
145 | # Test backfill with unfound object | |
146 | function TEST_rep_backfill_unfound() { | |
147 | local dir=$1 | |
148 | local objname=myobject | |
149 | local lastobj=300 | |
150 | # Must be between 1 and $lastobj | |
151 | local testobj=obj250 | |
152 | ||
153 | export CEPH_ARGS | |
154 | CEPH_ARGS+=' --osd_min_pg_log_entries=5 --osd_max_pg_log_entries=10' | |
155 | setup_osds 3 || return 1 | |
156 | ||
157 | local poolname=test-pool | |
158 | create_pool $poolname 1 1 || return 1 | |
159 | wait_for_clean || return 1 | |
160 | ||
161 | ceph pg dump pgs | |
162 | ||
163 | rados_put $dir $poolname $objname || return 1 | |
164 | ||
165 | local -a initial_osds=($(get_osds $poolname $objname)) | |
166 | local last_osd=${initial_osds[-1]} | |
167 | kill_daemons $dir TERM osd.${last_osd} 2>&2 < /dev/null || return 1 | |
168 | ceph osd down ${last_osd} || return 1 | |
169 | ceph osd out ${last_osd} || return 1 | |
170 | ||
171 | ceph pg dump pgs | |
172 | ||
173 | dd if=/dev/urandom of=${dir}/ORIGINAL bs=1024 count=4 | |
174 | for i in $(seq 1 $lastobj) | |
175 | do | |
176 | rados --pool $poolname put obj${i} $dir/ORIGINAL || return 1 | |
177 | done | |
178 | ||
179 | inject_eio rep data $poolname $testobj $dir 0 || return 1 | |
180 | inject_eio rep data $poolname $testobj $dir 1 || return 1 | |
181 | ||
182 | run_osd $dir ${last_osd} || return 1 | |
183 | ceph osd in ${last_osd} || return 1 | |
184 | ||
185 | sleep 15 | |
186 | ||
187 | for tmp in $(seq 1 100); do | |
188 | state=$(get_state 2.0) | |
189 | echo $state | grep backfill_unfound | |
190 | if [ "$?" = "0" ]; then | |
191 | break | |
192 | fi | |
193 | echo "$state " | |
194 | sleep 1 | |
195 | done | |
196 | ||
197 | ceph pg dump pgs | |
198 | ceph pg 2.0 list_missing | grep -q $testobj || return 1 | |
199 | ||
200 | # Command should hang because object is unfound | |
201 | timeout 5 rados -p $poolname get $testobj $dir/CHECK | |
202 | test $? = "124" || return 1 | |
203 | ||
204 | ceph pg 2.0 mark_unfound_lost delete | |
205 | ||
206 | wait_for_clean || return 1 | |
207 | ||
208 | for i in $(seq 1 $lastobj) | |
209 | do | |
210 | if [ obj${i} = "$testobj" ]; then | |
211 | # Doesn't exist anymore | |
212 | ! rados -p $poolname get $testobj $dir/CHECK || return 1 | |
213 | else | |
214 | rados --pool $poolname get obj${i} $dir/CHECK || return 1 | |
215 | diff -q $dir/ORIGINAL $dir/CHECK || return 1 | |
216 | fi | |
217 | done | |
218 | ||
219 | rm -f ${dir}/ORIGINAL ${dir}/CHECK | |
220 | ||
221 | delete_pool $poolname | |
222 | } | |
223 | ||
224 | # Test recovery with unfound object | |
225 | function TEST_rep_recovery_unfound() { | |
226 | local dir=$1 | |
227 | local objname=myobject | |
228 | local lastobj=100 | |
229 | # Must be between 1 and $lastobj | |
230 | local testobj=obj75 | |
231 | ||
232 | setup_osds 3 || return 1 | |
233 | ||
234 | local poolname=test-pool | |
235 | create_pool $poolname 1 1 || return 1 | |
236 | wait_for_clean || return 1 | |
237 | ||
238 | ceph pg dump pgs | |
239 | ||
240 | rados_put $dir $poolname $objname || return 1 | |
241 | ||
242 | local -a initial_osds=($(get_osds $poolname $objname)) | |
243 | local last_osd=${initial_osds[-1]} | |
244 | kill_daemons $dir TERM osd.${last_osd} 2>&2 < /dev/null || return 1 | |
245 | ceph osd down ${last_osd} || return 1 | |
246 | ceph osd out ${last_osd} || return 1 | |
247 | ||
248 | ceph pg dump pgs | |
249 | ||
250 | dd if=/dev/urandom of=${dir}/ORIGINAL bs=1024 count=4 | |
251 | for i in $(seq 1 $lastobj) | |
252 | do | |
253 | rados --pool $poolname put obj${i} $dir/ORIGINAL || return 1 | |
254 | done | |
255 | ||
256 | inject_eio rep data $poolname $testobj $dir 0 || return 1 | |
257 | inject_eio rep data $poolname $testobj $dir 1 || return 1 | |
258 | ||
259 | run_osd $dir ${last_osd} || return 1 | |
260 | ceph osd in ${last_osd} || return 1 | |
261 | ||
262 | sleep 15 | |
263 | ||
264 | for tmp in $(seq 1 100); do | |
265 | state=$(get_state 2.0) | |
266 | echo $state | grep -v recovering | |
267 | if [ "$?" = "0" ]; then | |
268 | break | |
269 | fi | |
270 | echo "$state " | |
271 | sleep 1 | |
272 | done | |
273 | ||
274 | ceph pg dump pgs | |
275 | ceph pg 2.0 list_missing | grep -q $testobj || return 1 | |
276 | ||
277 | # Command should hang because object is unfound | |
278 | timeout 5 rados -p $poolname get $testobj $dir/CHECK | |
279 | test $? = "124" || return 1 | |
280 | ||
281 | ceph pg 2.0 mark_unfound_lost delete | |
282 | ||
283 | wait_for_clean || return 1 | |
284 | ||
285 | for i in $(seq 1 $lastobj) | |
286 | do | |
287 | if [ obj${i} = "$testobj" ]; then | |
288 | # Doesn't exist anymore | |
289 | ! rados -p $poolname get $testobj $dir/CHECK || return 1 | |
290 | else | |
291 | rados --pool $poolname get obj${i} $dir/CHECK || return 1 | |
292 | diff -q $dir/ORIGINAL $dir/CHECK || return 1 | |
293 | fi | |
294 | done | |
295 | ||
296 | rm -f ${dir}/ORIGINAL ${dir}/CHECK | |
297 | ||
298 | delete_pool $poolname | |
299 | } | |
300 | ||
301 | main osd-rep-recov-eio.sh "$@" | |
302 | ||
303 | # Local Variables: | |
304 | # compile-command: "cd ../../../build ; make -j4 && ../qa/run-standalone.sh osd-rep-recov-eio.sh" | |
305 | # End: |