3 # Copyright (C) 2018 Red Hat <contact@redhat.com>
5 # Author: David Zafman <dzafman@redhat.com>
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU Library Public License as published by
9 # the Free Software Foundation; either version 2, or (at your option)
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Library Public License for more details.
18 source $CEPH_ROOT/qa
/standalone
/ceph-helpers.sh
24 export CEPH_MON
="127.0.0.1:7180" # git grep '\<7180\>' : there must be only one
26 CEPH_ARGS
+="--fsid=$(uuidgen) --auth-supported=none "
27 CEPH_ARGS
+="--mon-host=$CEPH_MON "
28 CEPH_ARGS
+="--osd_min_pg_log_entries=5 --osd_max_pg_log_entries=10 "
29 CEPH_ARGS
+="--fake_statfs_for_testing=3686400 "
30 CEPH_ARGS
+="--osd_max_backfills=10 "
32 export poolprefix
=test
34 local funcs
=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
35 for func
in $funcs ; do
36 setup
$dir ||
return 1
37 $func $dir ||
return 1
38 teardown
$dir ||
return 1
43 function get_num_in_state
() {
46 expression
+="select(contains(\"${state}\"))"
47 ceph
--format json pg dump pgs
2>/dev
/null | \
48 jq
".pg_stats | [.[] | .state | $expression] | length"
52 function wait_for_state
() {
56 local -a delays
=($
(get_timeout_delays
$2 5))
59 flush_pg_stats ||
return 1
60 while test $
(get_num_pgs
) == 0 ; do
65 cur_in_state
=$
(get_num_in_state
${state})
66 test $cur_in_state = "0" && break
67 if test $cur_in_state != $num_in_state ; then
69 num_in_state
=$cur_in_state
70 elif (( $loop >= ${#delays[*]} )) ; then
74 sleep ${delays[$loop]}
81 function wait_for_backfill
() {
83 wait_for_state backfilling
$timeout
87 function wait_for_active
() {
89 wait_for_state activating
$timeout
92 # All tests are created in an environment which has fake total space
93 # of 3600K (3686400) which can hold 600 6K replicated objects or
94 # 200 18K shards of erasure coded objects. For a k=3, m=2 EC pool
95 # we have a theoretical 54K object but with the chunk size of 4K
96 # and a rounding of 4K to account for the chunks is 36K max object
97 # which is ((36K / 3) + 4K) * 200 = 3200K which is 88% of
100 # Create 2 pools with size 1
101 # Write enough data that only 1 pool pg can fit per osd
102 # Incresase the pool size to 2
103 # On 3 OSDs this should result in 1 OSD with overlapping replicas,
104 # so both pools can't fit. We assume pgid 1.0 and 2.0 won't
105 # map to the same 2 OSDs.
106 # At least 1 pool shouldn't have room to backfill
107 # All other pools should go active+clean
108 function TEST_backfill_test_simple
() {
113 run_mon
$dir a ||
return 1
114 run_mgr
$dir x ||
return 1
117 for osd
in $
(seq 0 $
(expr $OSDS - 1))
119 run_osd
$dir $osd ||
return 1
122 ceph osd set-backfillfull-ratio
.85
124 for p
in $
(seq 1 $pools)
126 create_pool
"${poolprefix}$p" 1 1
127 ceph osd pool
set "${poolprefix}$p" size
1
130 wait_for_clean ||
return 1
132 # This won't work is if the 2 pools primary and only osds
135 dd if=/dev
/urandom of
=$dir/datafile bs
=1024 count
=4
136 for o
in $
(seq 1 $objects)
138 for p
in $
(seq 1 $pools)
140 rados
-p "${poolprefix}$p" put obj
$o $dir/datafile
146 for p
in $
(seq 1 $pools)
148 ceph osd pool
set "${poolprefix}$p" size
2
152 wait_for_backfill
240 ||
return 1
153 wait_for_active
60 ||
return 1
156 if [ "$(ceph pg dump pgs | grep +backfill_toofull | wc -l)" != "1" ];
158 echo "One pool should have been in backfill_toofull"
159 ERRORS
="$(expr $ERRORS + 1)"
162 expected
="$(expr $pools - 1)"
163 if [ "$(ceph pg dump pgs | grep active+clean | wc -l)" != "$expected" ];
165 echo "$expected didn't finish backfill"
166 ERRORS
="$(expr $ERRORS + 1)"
171 if [ $ERRORS != "0" ];
176 for i
in $
(seq 1 $pools)
178 delete_pool
"${poolprefix}$i"
180 kill_daemons
$dir ||
return 1
181 ! grep -q "num_bytes mismatch" $dir/osd.
*.log ||
return 1
185 # Create 8 pools of size 1 on 20 OSDs
186 # Write 4K * 600 objects (only 1 pool pg can fit on any given osd)
187 # Increase pool size to 2
188 # At least 1 pool shouldn't have room to backfill
189 # All other pools should go active+clean
190 function TEST_backfill_test_multi
() {
195 run_mon
$dir a ||
return 1
196 run_mgr
$dir x ||
return 1
199 for osd
in $
(seq 0 $
(expr $OSDS - 1))
201 run_osd
$dir $osd ||
return 1
204 ceph osd set-backfillfull-ratio
.85
206 for p
in $
(seq 1 $pools)
208 create_pool
"${poolprefix}$p" 1 1
209 ceph osd pool
set "${poolprefix}$p" size
1
212 wait_for_clean ||
return 1
214 dd if=/dev
/urandom of
=$dir/datafile bs
=1024 count
=4
215 for o
in $
(seq 1 $objects)
217 for p
in $
(seq 1 $pools)
219 rados
-p "${poolprefix}$p" put obj
$o $dir/datafile
225 for p
in $
(seq 1 $pools)
227 ceph osd pool
set "${poolprefix}$p" size
2
231 wait_for_backfill
240 ||
return 1
232 wait_for_active
60 ||
return 1
235 full
="$(ceph pg dump pgs | grep +backfill_toofull | wc -l)"
236 if [ "$full" -lt "1" ];
238 echo "At least one pool should have been in backfill_toofull"
239 ERRORS
="$(expr $ERRORS + 1)"
242 expected
="$(expr $pools - $full)"
243 if [ "$(ceph pg dump pgs | grep active+clean | wc -l)" != "$expected" ];
245 echo "$expected didn't finish backfill"
246 ERRORS
="$(expr $ERRORS + 1)"
251 if [ $ERRORS != "0" ];
256 for i
in $
(seq 1 $pools)
258 delete_pool
"${poolprefix}$i"
260 # Work around for http://tracker.ceph.com/issues/38195
261 kill_daemons
$dir #|| return 1
262 ! grep -q "num_bytes mismatch" $dir/osd.
*.log ||
return 1
266 # To make sure that when 2 pg try to backfill at the same time to
267 # the same target. This might be covered by the simple test above
268 # but this makes sure we get it.
270 # Create 10 pools of size 2 and identify 2 that have the same
272 # Delete all other pools
273 # Set size to 1 and write 4K * 600 to each pool
275 # The 2 pools should race to backfill.
276 # One pool goes active+clean
277 # The other goes acitve+...+backfill_toofull
278 function TEST_backfill_test_sametarget
() {
283 run_mon
$dir a ||
return 1
284 run_mgr
$dir x ||
return 1
287 for osd
in $
(seq 0 $
(expr $OSDS - 1))
289 run_osd
$dir $osd ||
return 1
292 ceph osd set-backfillfull-ratio
.85
294 for p
in $
(seq 1 $pools)
296 create_pool
"${poolprefix}$p" 1 1
297 ceph osd pool
set "${poolprefix}$p" size
2
301 wait_for_clean ||
return 1
305 # Find 2 pools with a pg that distinct primaries but second
306 # replica on the same osd.
316 for p
in $
(seq 1 $pools)
318 ceph pg map
${p}.0 --format=json | jq
'.acting[]' > $dir/acting
319 local test_osd1
=$
(head -1 $dir/acting
)
320 local test_osd2
=$
(tail -1 $dir/acting
)
325 pool1
="${poolprefix}$p"
328 elif [ $chk_osd1 != $test_osd1 -a $chk_osd2 = $test_osd2 ];
332 pool2
="${poolprefix}$p"
338 if [ "$pool2" = "" ];
340 echo "Failure to find appropirate PGs"
344 for p
in $
(seq 1 $pools)
346 if [ $p != $POOLNUM1 -a $p != $POOLNUM2 ];
348 delete_pool
${poolprefix}$p
352 ceph osd pool
set $pool1 size
1
353 ceph osd pool
set $pool2 size
1
355 wait_for_clean ||
return 1
357 dd if=/dev
/urandom of
=$dir/datafile bs
=1024 count
=4
358 for i
in $
(seq 1 $objects)
360 rados
-p $pool1 put obj
$i $dir/datafile
361 rados
-p $pool2 put obj
$i $dir/datafile
364 ceph osd pool
set $pool1 size
2
365 ceph osd pool
set $pool2 size
2
368 wait_for_backfill
240 ||
return 1
369 wait_for_active
60 ||
return 1
372 if [ "$(ceph pg dump pgs | grep +backfill_toofull | wc -l)" != "1" ];
374 echo "One pool should have been in backfill_toofull"
375 ERRORS
="$(expr $ERRORS + 1)"
378 if [ "$(ceph pg dump pgs | grep active+clean | wc -l)" != "1" ];
380 echo "One didn't finish backfill"
381 ERRORS
="$(expr $ERRORS + 1)"
386 if [ $ERRORS != "0" ];
393 kill_daemons
$dir ||
return 1
394 ! grep -q "num_bytes mismatch" $dir/osd.
*.log ||
return 1
397 # 2 pools can't both backfill to a target which has other data
398 # 1 of the pools has objects that increase from 1024 to 2611 bytes
400 # Write to fill pool which is size 1
401 # Take fill pool osd down (other 2 pools must go to the remaining OSDs
402 # Save an export of data on fill OSD and restart it
403 # Write an intial 1K to pool1 which has pg 2.0
404 # Export 2.0 from non-fillpool OSD don't wait for it to start-up
405 # Take down fillpool OSD
406 # Put 1K object version of 2.0 on fillpool OSD
407 # Put back fillpool data on fillpool OSD
408 # With fillpool down write 2611 byte objects
409 # Take down $osd and bring back $fillosd simultaneously
410 # Wait for backfilling
411 # One PG will be able to backfill its remaining data
412 # One PG must get backfill_toofull
413 function TEST_backfill_multi_partial
() {
419 run_mon
$dir a ||
return 1
420 run_mgr
$dir x ||
return 1
423 for osd
in $
(seq 0 $
(expr $OSDS - 1))
425 run_osd
$dir $osd ||
return 1
428 ceph osd set-backfillfull-ratio
.85
430 ceph osd set-require-min-compat-client luminous
431 create_pool fillpool
1 1
432 ceph osd pool
set fillpool size
1
433 for p
in $
(seq 1 $pools)
435 create_pool
"${poolprefix}$p" 1 1
436 ceph osd pool
set "${poolprefix}$p" size
2
439 wait_for_clean ||
return 1
441 # Partially fill an osd
442 # We have room for 600 6K replicated objects, if we create 2611 byte objects
443 # there is 3600K - (2611 * 600) = 2070K, so the fill pool and one
444 # replica from the other 2 is 85% of 3600K
446 dd if=/dev
/urandom of
=$dir/datafile bs
=2611 count
=1
447 for o
in $
(seq 1 $objects)
449 rados
-p fillpool put obj-fill-
${o} $dir/datafile
452 local fillosd
=$
(get_primary fillpool obj-fill-1
)
453 osd
=$
(expr $fillosd + 1)
454 if [ "$osd" = "$OSDS" ]; then
459 kill $
(cat $dir/osd.
$fillosd.pid
)
460 ceph osd out osd.
$fillosd
463 _objectstore_tool_nodown
$dir $fillosd --op export-remove
--pgid 1.0 --file $dir/fillexport.out ||
return 1
464 activate_osd
$dir $fillosd ||
return 1
468 dd if=/dev
/urandom of
=$dir/datafile bs
=1024 count
=1
469 for o
in $
(seq 1 $objects)
471 rados
-p "${poolprefix}1" put obj-1-
${o} $dir/datafile
475 # The $osd OSD is started, but we don't wait so we can kill $fillosd at the same time
476 _objectstore_tool_nowait
$dir $osd --op export --pgid 2.0 --file $dir/export.out
477 kill $
(cat $dir/osd.
$fillosd.pid
)
479 _objectstore_tool_nodown
$dir $fillosd --force --op remove
--pgid 2.0
480 _objectstore_tool_nodown
$dir $fillosd --op import
--pgid 2.0 --file $dir/export.out ||
return 1
481 _objectstore_tool_nodown
$dir $fillosd --op import
--pgid 1.0 --file $dir/fillexport.out ||
return 1
486 # re-write everything
487 dd if=/dev
/urandom of
=$dir/datafile bs
=2611 count
=1
488 for o
in $
(seq 1 $objects)
490 for p
in $
(seq 1 $pools)
492 rados
-p "${poolprefix}$p" put obj-${p}-${o} $dir/datafile
496 kill $(cat $dir/osd.$osd.pid)
497 ceph osd out osd.$osd
499 activate_osd $dir $fillosd || return 1
500 ceph osd in osd.$fillosd
503 wait_for_backfill 240 || return 1
504 wait_for_active 60 || return 1
506 flush_pg_stats || return 1
510 if [ "$
(get_num_in_state backfill_toofull
)" != "1" ];
512 echo "One PG should be
in backfill_toofull
"
513 ERRORS="$
(expr $ERRORS + 1)"
516 if [ "$
(get_num_in_state active
+clean
)" != "2" ];
518 echo "Two PGs should be active
+clean after one PG completed backfill
"
519 ERRORS="$
(expr $ERRORS + 1)"
522 if [ $ERRORS != "0" ];
528 for i in $(seq 1 $pools)
530 delete_pool "${poolprefix}$i"
532 kill_daemons $dir || return 1
533 ! grep -q "num_bytes mismatch
" $dir/osd.*.log || return 1
536 # Make sure that the amount of bytes already on the replica doesn't
537 # cause an out of space condition
539 # Create 1 pool and write 4K * 600 objects
540 # Remove 25% (150) of the objects with one OSD down (noout set)
541 # Increase the size of the remaining 75% (450) of the objects to 6K
542 # Bring back down OSD
543 # The pool should go active+clean
544 function TEST_backfill_grow() {
546 local poolname="test"
549 run_mon $dir a || return 1
550 run_mgr $dir x || return 1
552 for osd in $(seq 0 $(expr $OSDS - 1))
554 run_osd $dir $osd || return 1
557 ceph osd set-backfillfull-ratio .85
559 create_pool $poolname 1 1
560 ceph osd pool set $poolname size 3
563 wait_for_clean || return 1
565 dd if=/dev/urandom of=${dir}/4kdata bs=1k count=4
566 for i in $(seq 1 $objects)
568 rados -p $poolname put obj$i $dir/4kdata
571 local PG=$(get_pg $poolname obj1)
572 # Remember primary during the backfill
573 local primary=$(get_primary $poolname obj1)
574 local otherosd=$(get_not_primary $poolname obj1)
577 kill_daemons $dir TERM $otherosd || return 1
579 rmobjects=$(expr $objects / 4)
580 for i in $(seq 1 $rmobjects)
582 rados -p $poolname rm obj$i
585 dd if=/dev/urandom of=${dir}/6kdata bs=6k count=1
586 for i in $(seq $(expr $rmobjects + 1) $objects)
588 rados -p $poolname put obj$i $dir/6kdata
591 activate_osd $dir $otherosd || return 1
593 ceph tell osd.$primary debug kick_recovery_wq 0
597 wait_for_clean || return 1
599 delete_pool $poolname
600 kill_daemons $dir || return 1
601 ! grep -q "num_bytes mismatch
" $dir/osd.*.log || return 1
604 # Create a 5 shard EC pool on 6 OSD cluster
605 # Fill 1 OSD with 2600K of data take that osd down.
606 # Write the EC pool on 5 OSDs
607 # Take down 1 (must contain an EC shard)
608 # Bring up OSD with fill data
609 # Not enought room to backfill to partially full OSD
610 function TEST_ec_backfill_simple() {
617 local ecobjects=$(expr $objects / $k)
619 run_mon $dir a || return 1
620 run_mgr $dir x || return 1
623 for osd in $(seq 0 $(expr $OSDS - 1))
625 run_osd $dir $osd || return 1
628 ceph osd set-backfillfull-ratio .85
629 create_pool fillpool 1 1
630 ceph osd pool set fillpool size 1
632 # Partially fill an osd
633 # We have room for 200 18K replicated objects, if we create 13K objects
634 # there is only 3600K - (13K * 200) = 1000K which won't hold
635 # a k=3 shard below ((18K / 3) + 4K) * 200 = 2000K
636 # Actual usage per shard is 8K * 200 = 1600K because 18K/3 is 6K which
637 # rounds to 8K. The 2000K is the ceiling on the 18K * 200 = 3600K logical
639 dd if=/dev/urandom of=$dir/datafile bs=1024 count=13
640 for o in $(seq 1 $ecobjects)
642 rados -p fillpool put obj$o $dir/datafile
645 local fillosd=$(get_primary fillpool obj1)
646 osd=$(expr $fillosd + 1)
647 if [ "$osd" = "$OSDS" ]; then
652 kill $(cat $dir/osd.$fillosd.pid)
653 ceph osd out osd.$fillosd
655 ceph osd erasure-code-profile set ec-profile k=$k m=$m crush-failure-domain=osd technique=reed_sol_van plugin=jerasure || return 1
657 for p in $(seq 1 $pools)
659 ceph osd pool create "${poolprefix}$p" 1 1 erasure ec-profile
662 # Can't wait for clean here because we created a stale pg
663 #wait_for_clean || return 1
668 dd if=/dev/urandom of=$dir/datafile bs=1024 count=18
669 for o in $(seq 1 $ecobjects)
671 for p in $(seq 1 $pools)
673 rados -p "${poolprefix}$p" put obj$o $dir/datafile
677 kill $(cat $dir/osd.$osd.pid)
678 ceph osd out osd.$osd
680 activate_osd $dir $fillosd || return 1
681 ceph osd in osd.$fillosd
686 wait_for_backfill 240 || return 1
687 wait_for_active 60 || return 1
692 if [ "$
(ceph pg dump pgs |
grep -v "^1.0" |
grep +backfill_toofull |
wc -l)" != "1" ]; then
693 echo "One pool should have been
in backfill_toofull
"
694 ERRORS="$
(expr $ERRORS + 1)"
697 if [ $ERRORS != "0" ];
703 for i in $(seq 1 $pools)
705 delete_pool "${poolprefix}$i"
707 kill_daemons $dir || return 1
715 for osd in $(seq 0 $(expr $OSDS - 1))
717 if [ $osd = $excludeosd ];
721 if [ -n "$osds" ]; then
729 # Create a pool with size 1 and fill with data so that only 1 EC shard can fit.
730 # Write data to 2 EC pools mapped to the same OSDs (excluding filled one)
731 # Remap the last OSD to partially full OSD on both pools
732 # The 2 pools should race to backfill.
733 # One pool goes active+clean
734 # The other goes acitve+...+backfill_toofull
735 function TEST_ec_backfill_multi() {
742 local ecobjects=$(expr $objects / $k)
744 run_mon $dir a || return 1
745 run_mgr $dir x || return 1
748 for osd in $(seq 0 $(expr $OSDS - 1))
750 run_osd $dir $osd || return 1
753 # This test requires that shards from 2 different pools
754 # fit on a given OSD, but both will not fix. I'm using
755 # making the fillosd plus 1 shard use 75% of the space,
756 # leaving not enough to be under the 85% set here.
757 ceph osd set-backfillfull-ratio .85
759 ceph osd set-require-min-compat-client luminous
760 create_pool fillpool 1 1
761 ceph osd pool set fillpool size 1
763 # Partially fill an osd
764 # We have room for 200 18K replicated objects, if we create 9K objects
765 # there is only 3600K - (9K * 200) = 1800K which will only hold
766 # one k=3 shard below ((12K / 3) + 4K) * 200 = 1600K
767 # The actual data will be (12K / 3) * 200 = 800K because the extra
768 # is the reservation padding for chunking.
769 dd if=/dev/urandom of=$dir/datafile bs=1024 count=9
770 for o in $(seq 1 $ecobjects)
772 rados -p fillpool put obj$o $dir/datafile
775 local fillosd=$(get_primary fillpool obj1)
776 ceph osd erasure-code-profile set ec-profile k=3 m=2 crush-failure-domain=osd technique=reed_sol_van plugin=jerasure || return 1
778 nonfillosds="$
(osdlist
$OSDS $fillosd)"
780 for p in $(seq 1 $pools)
782 ceph osd pool create "${poolprefix}$p" 1 1 erasure ec-profile
783 ceph osd pg-upmap "$
(expr $p + 1).0" $nonfillosds
786 # Can't wait for clean here because we created a stale pg
787 #wait_for_clean || return 1
792 dd if=/dev/urandom of=$dir/datafile bs=1024 count=12
793 for o in $(seq 1 $ecobjects)
795 for p in $(seq 1 $pools)
797 rados -p "${poolprefix}$p" put obj$o-$p $dir/datafile
803 for p in $(seq 1 $pools)
805 ceph osd pg-upmap $(expr $p + 1).0 ${nonfillosds% *} $fillosd
810 wait_for_backfill 240 || return 1
811 wait_for_active 60 || return 1
816 if [ "$
(ceph pg dump pgs |
grep -v "^1.0" |
grep +backfill_toofull |
wc -l)" != "1" ];
818 echo "One pool should have been
in backfill_toofull
"
819 ERRORS="$
(expr $ERRORS + 1)"
822 if [ "$
(ceph pg dump pgs |
grep -v "^1.0" |
grep active
+clean |
wc -l)" != "1" ];
824 echo "One didn
't finish backfill"
825 ERRORS="$(expr $ERRORS + 1)"
828 if [ $ERRORS != "0" ];
834 for i in $(seq 1 $pools)
836 delete_pool "${poolprefix}$i"
838 kill_daemons $dir || return 1
841 # Similar to TEST_ec_backfill_multi but one of the ec pools
842 # already had some data on the target OSD
844 # Create a pool with size 1 and fill with data so that only 1 EC shard can fit.
845 # Write a small amount of data to 1 EC pool that still includes the filled one
846 # Take down fillosd with noout set
847 # Write data to 2 EC pools mapped to the same OSDs (excluding filled one)
848 # Remap the last OSD to partially full OSD on both pools
849 # The 2 pools should race to backfill.
850 # One pool goes active+clean
851 # The other goes acitve+...+backfill_toofull
852 function SKIP_TEST_ec_backfill_multi_partial() {
859 local ecobjects=$(expr $objects / $k)
860 local lastosd=$(expr $OSDS - 1)
862 run_mon $dir a || return 1
863 run_mgr $dir x || return 1
866 for osd in $(seq 0 $(expr $OSDS - 1))
868 run_osd $dir $osd || return 1
871 # This test requires that shards from 2 different pools
872 # fit on a given OSD, but both will not fix. I'm using
873 # making the fillosd plus 1 shard use 75% of the space,
874 # leaving not enough to be under the 85% set here.
875 ceph osd set-backfillfull-ratio
.85
877 ceph osd set-require-min-compat-client luminous
878 create_pool fillpool
1 1
879 ceph osd pool
set fillpool size
1
881 ceph osd pg-upmap
1.0 $lastosd
883 # Partially fill an osd
884 # We have room for 200 18K replicated objects, if we create 9K objects
885 # there is only 3600K - (9K * 200) = 1800K which will only hold
886 # one k=3 shard below ((12K / 3) + 4K) * 200 = 1600K
887 # The actual data will be (12K / 3) * 200 = 800K because the extra
888 # is the reservation padding for chunking.
889 dd if=/dev
/urandom of
=$dir/datafile bs
=1024 count
=9
890 for o
in $
(seq 1 $ecobjects)
892 rados
-p fillpool put obj
$o $dir/datafile
895 local fillosd
=$
(get_primary fillpool obj1
)
896 ceph osd erasure-code-profile
set ec-profile k
=3 m
=2 crush-failure-domain
=osd technique
=reed_sol_van plugin
=jerasure ||
return 1
898 nonfillosds
="$(osdlist $OSDS $fillosd)"
900 for p
in $
(seq 1 $pools)
902 ceph osd pool create
"${poolprefix}$p" 1 1 erasure ec-profile
903 ceph osd pg-upmap
"$(expr $p + 1).0" $
(seq 0 $lastosd)
906 # Can't wait for clean here because we created a stale pg
907 #wait_for_clean || return 1
912 dd if=/dev
/urandom of
=$dir/datafile bs
=1024 count
=1
913 for o
in $
(seq 1 $ecobjects)
915 rados
-p "${poolprefix}1" put obj
$o-1 $dir/datafile
918 for p
in $
(seq 1 $pools)
920 ceph osd pg-upmap
"$(expr $p + 1).0" $
(seq 0 $
(expr $lastosd - 1))
925 #kill_daemons $dir TERM osd.$lastosd || return 1
927 dd if=/dev
/urandom of
=$dir/datafile bs
=1024 count
=12
928 for o
in $
(seq 1 $ecobjects)
930 for p
in $
(seq 1 $pools)
932 rados
-p "${poolprefix}$p" put obj
$o-$p $dir/datafile
938 # Now backfill lastosd by adding back into the upmap
939 for p
in $
(seq 1 $pools)
941 ceph osd pg-upmap
"$(expr $p + 1).0" $
(seq 0 $lastosd)
943 #activate_osd $dir $lastosd || return 1
944 #ceph tell osd.0 debug kick_recovery_wq 0
949 wait_for_backfill
240 ||
return 1
950 wait_for_active
60 ||
return 1
955 if [ "$(ceph pg dump pgs | grep -v "^
1.0" | grep +backfill_toofull | wc -l)" != "1" ];
957 echo "One pool should have been in backfill_toofull"
958 ERRORS
="$(expr $ERRORS + 1)"
961 if [ "$(ceph pg dump pgs | grep -v "^
1.0" | grep active+clean | wc -l)" != "1" ];
963 echo "One didn't finish backfill"
964 ERRORS
="$(expr $ERRORS + 1)"
967 if [ $ERRORS != "0" ];
973 for i
in $
(seq 1 $pools)
975 delete_pool
"${poolprefix}$i"
977 kill_daemons
$dir ||
return 1
980 function SKIP_TEST_ec_backfill_multi_partial
() {
986 run_mon
$dir a ||
return 1
987 run_mgr
$dir x ||
return 1
990 for osd
in $
(seq 0 $
(expr $OSDS - 1))
992 run_osd
$dir $osd ||
return 1
995 # Below we need to fit 3200K in 3600K which is 88%
997 ceph osd set-backfillfull-ratio
.90
999 ceph osd set-require-min-compat-client luminous
1000 create_pool fillpool
1 1
1001 ceph osd pool
set fillpool size
1
1003 # Partially fill an osd
1004 # We have room for 200 48K ec objects, if we create 4k replicated objects
1005 # there is 3600K - (4K * 200) = 2800K which won't hold 2 k=3 shard
1006 # of 200 12K objects which takes ((12K / 3) + 4K) * 200 = 1600K each.
1007 # On the other OSDs 2 * 1600K = 3200K which is 88% of 3600K.
1008 dd if=/dev
/urandom of
=$dir/datafile bs
=1024 count
=4
1009 for o
in $
(seq 1 $objects)
1011 rados
-p fillpool put obj
$o $dir/datafile
1014 local fillosd
=$
(get_primary fillpool obj1
)
1015 osd
=$
(expr $fillosd + 1)
1016 if [ "$osd" = "$OSDS" ]; then
1021 kill $
(cat $dir/osd.
$fillosd.pid
)
1022 ceph osd out osd.
$fillosd
1024 ceph osd erasure-code-profile
set ec-profile k
=3 m
=2 crush-failure-domain
=osd technique
=reed_sol_van plugin
=jerasure ||
return 1
1026 for p
in $
(seq 1 $pools)
1028 ceph osd pool create
"${poolprefix}$p" 1 1 erasure ec-profile
1031 # Can't wait for clean here because we created a stale pg
1032 #wait_for_clean || return 1
1037 dd if=/dev
/urandom of
=$dir/datafile bs
=1024 count
=12
1038 for o
in $
(seq 1 $objects)
1040 for p
in $
(seq 1 $pools)
1042 rados
-p "${poolprefix}$p" put obj
$o $dir/datafile
1046 #ceph pg map 2.0 --format=json | jq '.'
1047 kill $
(cat $dir/osd.
$osd.pid
)
1048 ceph osd out osd.
$osd
1050 _objectstore_tool_nodown
$dir $osd --op export --pgid 2.0 --file $dir/export.out
1051 _objectstore_tool_nodown
$dir $fillosd --op import
--pgid 2.0 --file $dir/export.out
1053 activate_osd
$dir $fillosd ||
return 1
1054 ceph osd
in osd.
$fillosd
1057 wait_for_backfill
240 ||
return 1
1058 wait_for_active
60 ||
return 1
1061 if [ "$(ceph pg dump pgs | grep -v "^
1.0" | grep +backfill_toofull | wc -l)" != "1" ];
1063 echo "One pool should have been in backfill_toofull"
1064 ERRORS
="$(expr $ERRORS + 1)"
1067 if [ "$(ceph pg dump pgs | grep -v "^
1.0" | grep active+clean | wc -l)" != "1" ];
1069 echo "One didn't finish backfill"
1070 ERRORS
="$(expr $ERRORS + 1)"
1075 if [ $ERRORS != "0" ];
1080 delete_pool fillpool
1081 for i
in $
(seq 1 $pools)
1083 delete_pool
"${poolprefix}$i"
1085 kill_daemons
$dir ||
return 1
1089 # Write 200 12K objects ((12K / 3) + 4K) *200) = 1600K
1090 # Take 1 shard's OSD down (with noout set)
1091 # Remove 50 objects ((12K / 3) + 4k) * 50) = 400K
1092 # Write 150 36K objects (grow 150 objects) 2400K
1093 # But there is already 1600K usage so backfill
1094 # would be too full if it didn't account for existing data
1095 # Bring back down OSD so it must backfill
1096 # It should go active+clean taking into account data already there
1097 function TEST_ec_backfill_grow
() {
1099 local poolname
="test"
1103 local ecobjects
=$
(expr $objects / $k)
1105 run_mon
$dir a ||
return 1
1106 run_mgr
$dir x ||
return 1
1108 for osd
in $
(seq 0 $
(expr $OSDS - 1))
1110 run_osd
$dir $osd ||
return 1
1113 ceph osd set-backfillfull-ratio
.85
1115 ceph osd set-require-min-compat-client luminous
1116 ceph osd erasure-code-profile
set ec-profile k
=$k m
=$m crush-failure-domain
=osd technique
=reed_sol_van plugin
=jerasure ||
return 1
1117 ceph osd pool create
$poolname 1 1 erasure ec-profile
1119 wait_for_clean ||
return 1
1121 dd if=/dev
/urandom of
=${dir}/12kdata bs
=1k count
=12
1122 for i
in $
(seq 1 $ecobjects)
1124 rados
-p $poolname put obj
$i $dir/12kdata
1127 local PG
=$
(get_pg
$poolname obj1
)
1128 # Remember primary during the backfill
1129 local primary
=$
(get_primary
$poolname obj1
)
1130 local otherosd
=$
(get_not_primary
$poolname obj1
)
1133 kill_daemons
$dir TERM
$otherosd ||
return 1
1135 rmobjects
=$
(expr $ecobjects / 4)
1136 for i
in $
(seq 1 $rmobjects)
1138 rados
-p $poolname rm obj
$i
1141 dd if=/dev
/urandom of
=${dir}/36kdata bs
=1k count
=36
1142 for i
in $
(seq $
(expr $rmobjects + 1) $ecobjects)
1144 rados
-p $poolname put obj
$i $dir/36kdata
1147 activate_osd
$dir $otherosd ||
return 1
1149 ceph tell osd.
$primary debug kick_recovery_wq
0
1153 wait_for_clean ||
return 1
1155 delete_pool
$poolname
1156 kill_daemons
$dir ||
return 1
1159 main osd-backfill-space
"$@"
1162 # compile-command: "make -j4 && ../qa/run-standalone.sh osd-backfill-space.sh"