3 # Copyright (C) 2015 Red Hat <contact@redhat.com>
6 # Author: Kefu Chai <kchai@redhat.com>
8 # This program is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU Library Public License as published by
10 # the Free Software Foundation; either version 2, or (at your option)
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU Library Public License for more details.
19 source $CEPH_ROOT/qa
/standalone
/ceph-helpers.sh
25 export CEPH_MON
="127.0.0.1:7112" # git grep '\<7112\>' : there must be only one
27 CEPH_ARGS
+="--fsid=$(uuidgen) --auth-supported=none "
28 CEPH_ARGS
+="--mon-host=$CEPH_MON "
30 local funcs
=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
31 for func
in $funcs ; do
32 setup
$dir ||
return 1
33 run_mon
$dir a ||
return 1
34 run_mgr
$dir x ||
return 1
35 create_rbd_pool ||
return 1
37 # check that erasure code plugins are preloaded
38 CEPH_ARGS
='' ceph
--admin-daemon $
(get_asok_path mon.a
) log flush ||
return 1
39 grep 'load: jerasure.*lrc' $dir/mon.a.log ||
return 1
40 $func $dir ||
return 1
41 teardown
$dir ||
return 1
45 function setup_osds
() {
49 for id
in $
(seq 0 $
(expr $count - 1)) ; do
50 run_osd
$dir $id ||
return 1
52 wait_for_clean ||
return 1
54 # check that erasure code plugins are preloaded
55 CEPH_ARGS
='' ceph
--admin-daemon $
(get_asok_path osd
.0) log flush ||
return 1
56 grep 'load: jerasure.*lrc' $dir/osd
.0.log ||
return 1
59 function get_state
() {
62 ceph
--format json pg dump pgs
2>/dev
/null | \
63 jq
-r ".[] | select(.pgid==\"$pgid\") | .$sname"
66 function create_erasure_coded_pool
() {
74 ceph osd erasure-code-profile
set myprofile \
77 crush-failure-domain
=osd ||
return 1
78 create_pool
$poolname 1 1 erasure myprofile \
80 wait_for_clean ||
return 1
83 function delete_pool
() {
86 ceph osd pool delete
$poolname $poolname --yes-i-really-really-mean-it
87 ceph osd erasure-code-profile
rm myprofile
90 function rados_put
() {
93 local objname
=${3:-SOMETHING}
95 for marker
in AAA BBB CCCC DDDD
; do
96 printf "%*s" 1024 $marker
99 # get and put an object, compare they are equal
101 rados
--pool $poolname put
$objname $dir/ORIGINAL ||
return 1
104 function rados_get
() {
107 local objname
=${3:-SOMETHING}
108 local expect
=${4:-ok}
111 # Expect a failure to get object
113 if [ $expect = "fail" ];
115 ! rados
--pool $poolname get
$objname $dir/COPY
119 # get an object, compare with $dir/ORIGINAL
121 rados
--pool $poolname get
$objname $dir/COPY ||
return 1
122 diff $dir/ORIGINAL
$dir/COPY ||
return 1
127 function inject_remove
() {
141 local -a initial_osds
=($
(get_osds
$poolname $objname))
142 local osd_id
=${initial_osds[$shard_id]}
143 objectstore_tool
$dir $osd_id $objname remove ||
return 1
146 # Test with an inject error
147 function rados_put_get_data
() {
156 # inject eio to speificied shard
158 local poolname
=pool-jerasure
159 local objname
=obj-
$inject-$$
-$shard_id
160 rados_put
$dir $poolname $objname ||
return 1
161 inject_
$inject ec data
$poolname $objname $dir $shard_id ||
return 1
162 rados_get
$dir $poolname $objname ||
return 1
164 if [ "$arg" = "recovery" ];
167 # take out the last OSD used to store the object,
168 # bring it back, and check for clean PGs which means
169 # recovery didn't crash the primary.
171 local -a initial_osds
=($
(get_osds
$poolname $objname))
172 local last_osd
=${initial_osds[-1]}
174 kill_daemons
$dir TERM osd.
${last_osd} >&2 < /dev
/null ||
return 1
175 ceph osd out
${last_osd} ||
return 1
176 ! get_osds
$poolname $objname |
grep '\<'${last_osd}'\>' ||
return 1
177 ceph osd
in ${last_osd} ||
return 1
178 run_osd
$dir ${last_osd} ||
return 1
179 wait_for_clean ||
return 1
182 shard_id
=$
(expr $shard_id + 1)
183 inject_
$inject ec data
$poolname $objname $dir $shard_id ||
return 1
184 # Now 2 out of 3 shards get an error, so should fail
185 rados_get
$dir $poolname $objname fail ||
return 1
189 # Change the size of speificied shard
191 function set_size
() {
202 local poolname
=pool-jerasure
203 local -a initial_osds
=($
(get_osds
$poolname $objname))
204 local osd_id
=${initial_osds[$shard_id]}
206 if [ "$mode" = "add" ];
208 objectstore_tool
$dir $osd_id $objname get-bytes
$dir/CORRUPT ||
return 1
209 dd if=/dev
/urandom bs
=$bytes count
=1 >> $dir/CORRUPT
210 elif [ "$bytes" = "0" ];
214 dd if=/dev
/urandom bs
=$bytes count
=1 of
=$dir/CORRUPT
216 objectstore_tool
$dir $osd_id $objname set-bytes
$dir/CORRUPT ||
return 1
221 function rados_get_data_bad_size
() {
230 local poolname
=pool-jerasure
231 local objname
=obj-size-$$
-$shard_id-$bytes
232 rados_put
$dir $poolname $objname ||
return 1
234 # Change the size of speificied shard
236 set_size
$objname $dir $shard_id $bytes $mode ||
return 1
238 rados_get
$dir $poolname $objname ||
return 1
240 # Leave objname and modify another shard
241 shard_id
=$
(expr $shard_id + 1)
242 set_size
$objname $dir $shard_id $bytes $mode ||
return 1
243 rados_get
$dir $poolname $objname fail ||
return 1
248 # These two test cases try to validate the following behavior:
249 # For object on EC pool, if there is one shard having read error (
250 # either primary or replica), client can still read object.
252 # If 2 shards have read errors the client will get an error.
254 function TEST_rados_get_subread_eio_shard_0
() {
256 setup_osds
4 ||
return 1
258 local poolname
=pool-jerasure
259 create_erasure_coded_pool
$poolname 2 1 ||
return 1
260 # inject eio on primary OSD (0) and replica OSD (1)
262 rados_put_get_data eio
$dir $shard_id ||
return 1
263 delete_pool
$poolname
266 function TEST_rados_get_subread_eio_shard_1
() {
268 setup_osds
4 ||
return 1
270 local poolname
=pool-jerasure
271 create_erasure_coded_pool
$poolname 2 1 ||
return 1
272 # inject eio into replicas OSD (1) and OSD (2)
274 rados_put_get_data eio
$dir $shard_id ||
return 1
275 delete_pool
$poolname
278 # We don't remove the object from the primary because
279 # that just causes it to appear to be missing
281 function TEST_rados_get_subread_missing
() {
283 setup_osds
4 ||
return 1
285 local poolname
=pool-jerasure
286 create_erasure_coded_pool
$poolname 2 1 ||
return 1
287 # inject remove into replicas OSD (1) and OSD (2)
289 rados_put_get_data remove
$dir $shard_id ||
return 1
290 delete_pool
$poolname
295 # These two test cases try to validate that following behavior:
296 # For object on EC pool, if there is one shard which an incorrect
297 # size this will cause an internal read error, client can still read object.
299 # If 2 shards have incorrect size the client will get an error.
301 function TEST_rados_get_bad_size_shard_0
() {
303 setup_osds
4 ||
return 1
305 local poolname
=pool-jerasure
306 create_erasure_coded_pool
$poolname 2 1 ||
return 1
307 # Set incorrect size into primary OSD (0) and replica OSD (1)
309 rados_get_data_bad_size
$dir $shard_id 10 ||
return 1
310 rados_get_data_bad_size
$dir $shard_id 0 ||
return 1
311 rados_get_data_bad_size
$dir $shard_id 256 add ||
return 1
312 delete_pool
$poolname
315 function TEST_rados_get_bad_size_shard_1
() {
317 setup_osds
4 ||
return 1
319 local poolname
=pool-jerasure
320 create_erasure_coded_pool
$poolname 2 1 ||
return 1
321 # Set incorrect size into replicas OSD (1) and OSD (2)
323 rados_get_data_bad_size
$dir $shard_id 10 ||
return 1
324 rados_get_data_bad_size
$dir $shard_id 0 ||
return 1
325 rados_get_data_bad_size
$dir $shard_id 256 add ||
return 1
326 delete_pool
$poolname
329 function TEST_rados_get_with_subreadall_eio_shard_0
() {
333 setup_osds
4 ||
return 1
335 local poolname
=pool-jerasure
336 create_erasure_coded_pool
$poolname 2 1 ||
return 1
337 # inject eio on primary OSD (0)
338 rados_put_get_data eio
$dir $shard_id recovery ||
return 1
340 delete_pool
$poolname
343 function TEST_rados_get_with_subreadall_eio_shard_1
() {
347 setup_osds
4 ||
return 1
349 local poolname
=pool-jerasure
350 create_erasure_coded_pool
$poolname 2 1 ||
return 1
351 # inject eio on replica OSD (1)
352 rados_put_get_data eio
$dir $shard_id recovery ||
return 1
354 delete_pool
$poolname
357 # Test recovery the first k copies aren't all available
358 function TEST_ec_recovery_errors
() {
360 local objname
=myobject
362 setup_osds
7 ||
return 1
364 local poolname
=pool-jerasure
365 create_erasure_coded_pool
$poolname 3 2 ||
return 1
367 rados_put
$dir $poolname $objname ||
return 1
368 inject_eio ec data
$poolname $objname $dir 0 ||
return 1
370 local -a initial_osds
=($
(get_osds
$poolname $objname))
371 local last_osd
=${initial_osds[-1]}
373 kill_daemons
$dir TERM osd.
${last_osd} >&2 < /dev
/null ||
return 1
374 ceph osd down
${last_osd} ||
return 1
375 ceph osd out
${last_osd} ||
return 1
377 # Cluster should recover this object
378 wait_for_clean ||
return 1
380 delete_pool
$poolname
383 # Test backfill with unfound object
384 function TEST_ec_backfill_unfound
() {
386 local objname
=myobject
388 # Must be between 1 and $lastobj
392 CEPH_ARGS
+=' --osd_min_pg_log_entries=5 --osd_max_pg_log_entries=10'
393 setup_osds
5 ||
return 1
395 local poolname
=pool-jerasure
396 create_erasure_coded_pool
$poolname 3 2 ||
return 1
400 rados_put
$dir $poolname $objname ||
return 1
402 local -a initial_osds
=($
(get_osds
$poolname $objname))
403 local last_osd
=${initial_osds[-1]}
404 kill_daemons
$dir TERM osd.
${last_osd} 2>&2 < /dev
/null ||
return 1
405 ceph osd down
${last_osd} ||
return 1
406 ceph osd out
${last_osd} ||
return 1
410 dd if=/dev
/urandom of
=${dir}/ORIGINAL bs
=1024 count
=4
411 for i
in $
(seq 1 $lastobj)
413 rados
--pool $poolname put obj
${i} $dir/ORIGINAL ||
return 1
416 inject_eio ec data
$poolname $testobj $dir 0 ||
return 1
417 inject_eio ec data
$poolname $testobj $dir 1 ||
return 1
419 run_osd
$dir ${last_osd} ||
return 1
420 ceph osd
in ${last_osd} ||
return 1
424 for tmp
in $
(seq 1 100); do
425 state
=$
(get_state
2.0)
426 echo $state |
grep backfill_unfound
427 if [ "$?" = "0" ]; then
435 ceph pg
2.0 list_missing |
grep -q $testobj ||
return 1
437 # Command should hang because object is unfound
438 timeout
5 rados
-p $poolname get
$testobj $dir/CHECK
439 test $?
= "124" ||
return 1
441 ceph pg
2.0 mark_unfound_lost delete
443 wait_for_clean ||
return 1
445 for i
in $
(seq 1 $lastobj)
447 if [ obj
${i} = "$testobj" ]; then
448 # Doesn't exist anymore
449 ! rados
-p $poolname get
$testobj $dir/CHECK ||
return 1
451 rados
--pool $poolname get obj
${i} $dir/CHECK ||
return 1
452 diff -q $dir/ORIGINAL
$dir/CHECK ||
return 1
456 rm -f ${dir}/ORIGINAL
${dir}/CHECK
458 delete_pool
$poolname
461 # Test recovery with unfound object
462 function TEST_ec_recovery_unfound
() {
464 local objname
=myobject
466 # Must be between 1 and $lastobj
469 setup_osds
5 ||
return 1
471 local poolname
=pool-jerasure
472 create_erasure_coded_pool
$poolname 3 2 ||
return 1
476 rados_put
$dir $poolname $objname ||
return 1
478 local -a initial_osds
=($
(get_osds
$poolname $objname))
479 local last_osd
=${initial_osds[-1]}
480 kill_daemons
$dir TERM osd.
${last_osd} 2>&2 < /dev
/null ||
return 1
481 ceph osd down
${last_osd} ||
return 1
482 ceph osd out
${last_osd} ||
return 1
486 dd if=/dev
/urandom of
=${dir}/ORIGINAL bs
=1024 count
=4
487 for i
in $
(seq 1 $lastobj)
489 rados
--pool $poolname put obj
${i} $dir/ORIGINAL ||
return 1
492 inject_eio ec data
$poolname $testobj $dir 0 ||
return 1
493 inject_eio ec data
$poolname $testobj $dir 1 ||
return 1
495 run_osd
$dir ${last_osd} ||
return 1
496 ceph osd
in ${last_osd} ||
return 1
500 for tmp
in $
(seq 1 100); do
501 state
=$
(get_state
2.0)
502 echo $state |
grep recovery_unfound
503 if [ "$?" = "0" ]; then
511 ceph pg
2.0 list_missing |
grep -q $testobj ||
return 1
513 # Command should hang because object is unfound
514 timeout
5 rados
-p $poolname get
$testobj $dir/CHECK
515 test $?
= "124" ||
return 1
517 ceph pg
2.0 mark_unfound_lost delete
519 wait_for_clean ||
return 1
521 for i
in $
(seq 1 $lastobj)
523 if [ obj
${i} = "$testobj" ]; then
524 # Doesn't exist anymore
525 ! rados
-p $poolname get
$testobj $dir/CHECK ||
return 1
527 rados
--pool $poolname get obj
${i} $dir/CHECK ||
return 1
528 diff -q $dir/ORIGINAL
$dir/CHECK ||
return 1
532 rm -f ${dir}/ORIGINAL
${dir}/CHECK
534 delete_pool
$poolname
537 main test-erasure-eio
"$@"
540 # compile-command: "cd ../.. ; make -j4 && test/erasure-code/test-erasure-eio.sh"