3 # Copyright (C) 2017 Red Hat <contact@redhat.com>
5 # Author: David Zafman <dzafman@redhat.com>
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU Library Public License as published by
9 # the Free Software Foundation; either version 2, or (at your option)
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Library Public License for more details.
18 source $CEPH_ROOT/qa
/standalone
/ceph-helpers.sh
25 export CEPH_MON
="127.0.0.1:7114" # git grep '\<7114\>' : there must be only one
27 CEPH_ARGS
+="--fsid=$(uuidgen) --auth-supported=none "
28 CEPH_ARGS
+="--mon-host=$CEPH_MON "
29 CEPH_ARGS
+="--osd_min_pg_log_entries=5 --osd_max_pg_log_entries=10 "
34 local funcs
=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
35 for func
in $funcs ; do
36 setup
$dir ||
return 1
37 $func $dir ||
return 1
38 teardown
$dir ||
return 1
42 function below_margin
() {
47 return $
(( $check <= $target && $check >= $target - $margin ?
0 : 1 ))
50 function above_margin
() {
55 return $
(( $check >= $target && $check <= $target + $margin ?
0 : 1 ))
61 local degraded_start
=$3
63 local misplaced_start
=$5
64 local misplaced_end
=$6
68 if [ "$type" = "erasure" ];
73 UPACT
=$
(grep "pg[[]${PG}.*backfilling.*_update_calc_stats " $log |
tail -1 |
sed "s/.*[)] \([[][^ p]*\).*$/\1/")
75 # Check 3rd line at start because of false backfill starts
76 FIRST
=$
(grep "pg[[]${PG}.*backfilling.*_update_calc_stats degraded " $log | grep -F " ${UPACT}${addp}" |
head -1 |
sed "s/.* \([0-9]*\)$/\1/")
77 below_margin
$FIRST $degraded_start ||
return 1
78 LAST
=$
(grep "pg[[]${PG}.*backfilling.*_update_calc_stats degraded " $log |
tail -1 |
sed "s/.* \([0-9]*\)$/\1/")
79 above_margin
$LAST $degraded_end ||
return 1
81 # Check 3rd line at start because of false backfill starts
82 FIRST
=$
(grep "pg[[]${PG}.*backfilling.*_update_calc_stats misplaced " $log | grep -F " ${UPACT}${addp}" |
head -1 |
sed "s/.* \([0-9]*\)$/\1/")
83 below_margin
$FIRST $misplaced_start ||
return 1
84 LAST
=$
(grep "pg[[]${PG}.*backfilling.*_update_calc_stats misplaced " $log |
tail -1 |
sed "s/.* \([0-9]*\)$/\1/")
85 above_margin
$LAST $misplaced_end ||
return 1
90 # state: active+undersized+degraded+remapped+backfilling
92 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
93 # 1.0 500 0 1000 0 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:44:23.531466 22'500 26:617 [1,0,2] 1 [1] 1 0'0 2017-10-27 09:43:44.654882 0'0 2017-10-27 09:43:44.654882
94 function TEST_backfill_sizeup
() {
97 run_mon
$dir a ||
return 1
98 run_mgr
$dir x ||
return 1
100 run_osd
$dir 0 ||
return 1
101 run_osd
$dir 1 ||
return 1
102 run_osd
$dir 2 ||
return 1
103 run_osd
$dir 3 ||
return 1
104 run_osd
$dir 4 ||
return 1
105 run_osd
$dir 5 ||
return 1
107 create_pool
$poolname 1 1
108 ceph osd pool
set $poolname size
1
110 wait_for_clean ||
return 1
112 for i
in $
(seq 1 $objects)
114 rados
-p $poolname put obj
$i /dev
/null
117 ceph osd pool
set $poolname size
3
120 wait_for_clean ||
return 1
122 local primary
=$
(get_primary
$poolname obj1
)
123 local PG
=$
(get_pg
$poolname obj1
)
125 local log
=$
(grep -l +backfilling
$dir/osd.
*.log
)
126 test -n "$log" ||
return 1
128 local degraded
=$
(expr $objects \
* 2)
129 check
$PG $log $degraded 0 0 0 ||
return 1
131 delete_pool
$poolname
132 kill_daemons
$dir ||
return 1
140 # state: active+undersized+degraded+remapped+backfilling
142 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
143 # 1.0 500 0 1000 500 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:48:53.326849 22'500 26:603 [0,2,4] 0 [1] 1 0'0 2017-10-27 09:48:13.236253 0'0 2017-10-27 09:48:13.236253
144 function TEST_backfill_sizeup_out
() {
147 run_mon
$dir a ||
return 1
148 run_mgr
$dir x ||
return 1
149 run_osd
$dir 0 ||
return 1
150 run_osd
$dir 1 ||
return 1
151 run_osd
$dir 2 ||
return 1
152 run_osd
$dir 3 ||
return 1
153 run_osd
$dir 4 ||
return 1
154 run_osd
$dir 5 ||
return 1
156 create_pool
$poolname 1 1
157 ceph osd pool
set $poolname size
1
159 wait_for_clean ||
return 1
161 for i
in $
(seq 1 $objects)
163 rados
-p $poolname put obj
$i /dev
/null
166 local PG
=$
(get_pg
$poolname obj1
)
167 # Remember primary during the backfill
168 local primary
=$
(get_primary
$poolname obj1
)
170 ceph osd out osd.
$primary
171 ceph osd pool
set $poolname size
3
174 wait_for_clean ||
return 1
176 local log
=$
(grep -l +backfilling
$dir/osd.
*.log
)
177 test -n "$log" ||
return 1
178 local degraded
=$
(expr $objects \
* 2)
179 check
$PG $log $degraded 0 $objects 0 ||
return 1
181 delete_pool
$poolname
182 kill_daemons
$dir ||
return 1
186 # [1 0] -> [1,2]/[1,0]
188 # state: active+remapped+backfilling
190 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
191 # 1.0 500 0 0 500 0 0 100 100 active+remapped+backfilling 2017-10-27 09:51:18.800517 22'500 25:570 [1,2] 1 [1,0] 1 0'0 2017-10-27 09:50:40.441274 0'0 2017-10-27 09:50:40.441274
192 function TEST_backfill_out
() {
195 run_mon
$dir a ||
return 1
196 run_mgr
$dir x ||
return 1
197 run_osd
$dir 0 ||
return 1
198 run_osd
$dir 1 ||
return 1
199 run_osd
$dir 2 ||
return 1
200 run_osd
$dir 3 ||
return 1
201 run_osd
$dir 4 ||
return 1
202 run_osd
$dir 5 ||
return 1
204 create_pool
$poolname 1 1
205 ceph osd pool
set $poolname size
2
208 wait_for_clean ||
return 1
210 for i
in $
(seq 1 $objects)
212 rados
-p $poolname put obj
$i /dev
/null
215 local PG
=$
(get_pg
$poolname obj1
)
216 # Remember primary during the backfill
217 local primary
=$
(get_primary
$poolname obj1
)
219 ceph osd out osd.$
(get_not_primary
$poolname obj1
)
222 wait_for_clean ||
return 1
224 local log
=$
(grep -l +backfilling
$dir/osd.
*.log
)
225 test -n "$log" ||
return 1
227 check
$PG $log 0 0 $objects 0 ||
return 1
229 delete_pool
$poolname
230 kill_daemons
$dir ||
return 1
234 # [0, 1] -> [0, 2]/[0]
237 # state: active+undersized+degraded+remapped+backfilling
239 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
240 # 1.0 500 0 500 0 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:53:24.051091 22'500 27:719 [0,2] 0 [0] 0 0'0 2017-10-27 09:52:43.188368 0'0 2017-10-27 09:52:43.188368
241 function TEST_backfill_down_out
() {
244 run_mon
$dir a ||
return 1
245 run_mgr
$dir x ||
return 1
246 run_osd
$dir 0 ||
return 1
247 run_osd
$dir 1 ||
return 1
248 run_osd
$dir 2 ||
return 1
249 run_osd
$dir 3 ||
return 1
250 run_osd
$dir 4 ||
return 1
251 run_osd
$dir 5 ||
return 1
253 create_pool
$poolname 1 1
254 ceph osd pool
set $poolname size
2
257 wait_for_clean ||
return 1
259 for i
in $
(seq 1 $objects)
261 rados
-p $poolname put obj
$i /dev
/null
264 local PG
=$
(get_pg
$poolname obj1
)
265 # Remember primary during the backfill
266 local primary
=$
(get_primary
$poolname obj1
)
267 local otherosd
=$
(get_not_primary
$poolname obj1
)
269 kill $
(cat $dir/osd.
${otherosd}.pid
)
270 ceph osd down osd.
${otherosd}
271 ceph osd out osd.
${otherosd}
274 wait_for_clean ||
return 1
276 local log
=$
(grep -l +backfilling
$dir/osd.
*.log
)
277 test -n "$log" ||
return 1
279 check
$PG $log $objects 0 0 0 ||
return 1
281 delete_pool
$poolname
282 kill_daemons
$dir ||
return 1
286 # [1, 0] -> [2, 3, 4]
288 # misplaced 1000 -> 0
289 # state: active+undersized+degraded+remapped+backfilling
291 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
292 # 1.0 500 0 500 1000 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:55:50.375722 23'500 27:553 [2,4,3] 2 [1,0] 1 0'0 2017-10-27 09:55:10.230919 0'0 2017-10-27 09:55:10.230919
293 function TEST_backfill_out2
() {
296 run_mon
$dir a ||
return 1
297 run_mgr
$dir x ||
return 1
298 run_osd
$dir 0 ||
return 1
299 run_osd
$dir 1 ||
return 1
300 run_osd
$dir 2 ||
return 1
301 run_osd
$dir 3 ||
return 1
302 run_osd
$dir 4 ||
return 1
303 run_osd
$dir 5 ||
return 1
305 create_pool
$poolname 1 1
306 ceph osd pool
set $poolname size
2
309 wait_for_clean ||
return 1
311 for i
in $
(seq 1 $objects)
313 rados
-p $poolname put obj
$i /dev
/null
316 local PG
=$
(get_pg
$poolname obj1
)
317 # Remember primary during the backfill
318 local primary
=$
(get_primary
$poolname obj1
)
319 local otherosd
=$
(get_not_primary
$poolname obj1
)
321 ceph osd
set nobackfill
322 ceph osd pool
set $poolname size
3
323 ceph osd out osd.
${otherosd}
324 ceph osd out osd.
${primary}
325 ceph osd
unset nobackfill
326 ceph tell osd.$
(get_primary
$poolname obj1
) debug kick_recovery_wq
0
329 wait_for_clean ||
return 1
331 local log
=$
(grep -l +backfilling
$dir/osd.
*.log
)
332 test -n "$log" ||
return 1
333 local misplaced
=$
(expr $objects \
* 2)
335 check
$PG $log $objects 0 $misplaced 0 ||
return 1
337 delete_pool
$poolname
338 kill_daemons
$dir ||
return 1
342 # [0,1] -> [2,4,3]/[0,1]
344 # misplaced 1000 -> 500
345 # state ends at active+clean+remapped [2,4,3]/[2,4,3,0]
346 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
347 # 1.0 500 0 1000 1000 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-10-30 18:21:45.995149 19'500 23:1817 [2,4,3] 2 [0,1] 0 0'0 2017-10-30 18:21:05.109904 0'0 2017-10-30 18:21:05.109904
349 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
350 # 1.0 500 0 0 500 0 0 5 5 active+clean+remapped 2017-10-30 18:22:42.293730 19'500 25:2557 [2,4,3] 2 [2,4,3,0] 2 0'0 2017-10-30 18:21:05.109904 0'0 2017-10-30 18:21:05.109904
351 function TEST_backfill_sizeup4_allout
() {
354 run_mon
$dir a ||
return 1
355 run_mgr
$dir x ||
return 1
356 run_osd
$dir 0 ||
return 1
357 run_osd
$dir 1 ||
return 1
358 run_osd
$dir 2 ||
return 1
359 run_osd
$dir 3 ||
return 1
360 run_osd
$dir 4 ||
return 1
362 create_pool
$poolname 1 1
363 ceph osd pool
set $poolname size
2
365 wait_for_clean ||
return 1
367 for i
in $
(seq 1 $objects)
369 rados
-p $poolname put obj
$i /dev
/null
372 local PG
=$
(get_pg
$poolname obj1
)
373 # Remember primary during the backfill
374 local primary
=$
(get_primary
$poolname obj1
)
375 local otherosd
=$
(get_not_primary
$poolname obj1
)
377 ceph osd
set nobackfill
378 ceph osd out osd.
$otherosd
379 ceph osd out osd.
$primary
380 ceph osd pool
set $poolname size
4
381 ceph osd
unset nobackfill
382 ceph tell osd.$
(get_primary
$poolname obj1
) debug kick_recovery_wq
0
385 wait_for_clean ||
return 1
387 local log
=$
(grep -l +backfilling
$dir/osd.
*.log
)
388 test -n "$log" ||
return 1
389 local misdeg
=$
(expr $objects \
* 2)
390 check
$PG $log $misdeg 0 $misdeg $objects ||
return 1
392 delete_pool
$poolname
393 kill_daemons
$dir ||
return 1
397 # [1,2,0] -> [3]/[1,2]
398 # misplaced 1000 -> 500
399 # state ends at active+clean+remapped [3]/[3,1]
400 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
401 # 1.0 500 0 0 1000 0 0 100 100 active+remapped+backfilling 2017-11-28 19:13:56.092439 21'500 31:790 [3] 3 [1,2] 1 0'0 2017-11-28 19:13:28.698661 0'0 2017-11-28 19:13:28.698661
402 function TEST_backfill_remapped
() {
405 run_mon
$dir a ||
return 1
406 run_mgr
$dir x ||
return 1
407 run_osd
$dir 0 ||
return 1
408 run_osd
$dir 1 ||
return 1
409 run_osd
$dir 2 ||
return 1
410 run_osd
$dir 3 ||
return 1
412 create_pool
$poolname 1 1
413 ceph osd pool
set $poolname size
3
416 wait_for_clean ||
return 1
418 for i
in $
(seq 1 $objects)
420 rados
-p $poolname put obj
$i /dev
/null
423 local PG
=$
(get_pg
$poolname obj1
)
424 # Remember primary during the backfill
425 local primary
=$
(get_primary
$poolname obj1
)
426 local otherosd
=$
(get_not_primary
$poolname obj1
)
428 ceph osd
set nobackfill
429 ceph osd out osd.
${otherosd}
430 for i
in $
(get_osds
$poolname obj1
)
432 if [ $i = $primary -o $i = $otherosd ];
439 ceph osd out osd.
${primary}
440 ceph osd pool
set $poolname size
2
442 ceph osd
unset nobackfill
443 ceph tell osd.$
(get_primary
$poolname obj1
) debug kick_recovery_wq
0
446 wait_for_clean ||
return 1
448 local log
=$
(grep -l +backfilling
$dir/osd.
*.log
)
449 test -n "$log" ||
return 1
450 local misplaced
=$
(expr $objects \
* 2)
452 check
$PG $log 0 0 $misplaced $objects ||
return 1
454 delete_pool
$poolname
455 kill_daemons
$dir ||
return 1
458 # [1,0,2] -> [4,3,NONE]/[1,0,2]
459 # misplaced 1500 -> 500
460 # state ends at active+clean+remapped [4,3,NONE]/[4,3,2]
462 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
463 # 1.0 500 0 0 1500 0 0 100 100 active+degraded+remapped+backfilling 2017-10-31 16:53:39.467126 19'500 23:615 [4,3,NONE] 4 [1,0,2] 1 0'0 2017-10-31 16:52:59.624429 0'0 2017-10-31 16:52:59.624429
468 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
469 # 1.0 500 0 0 500 0 0 5 5 active+clean+remapped 2017-10-31 16:48:34.414040 19'500 25:2049 [4,3,NONE] 4 [4,3,2] 4 0'0 2017-10-31 16:46:58.203440 0'0 2017-10-31 16:46:58.203440
470 function TEST_backfill_ec_all_out
() {
473 run_mon
$dir a ||
return 1
474 run_mgr
$dir x ||
return 1
475 run_osd
$dir 0 ||
return 1
476 run_osd
$dir 1 ||
return 1
477 run_osd
$dir 2 ||
return 1
478 run_osd
$dir 3 ||
return 1
479 run_osd
$dir 4 ||
return 1
481 ceph osd erasure-code-profile
set myprofile plugin
=jerasure technique
=reed_sol_van k
=2 m
=1 crush-failure-domain
=osd
482 create_pool
$poolname 1 1 erasure myprofile
484 wait_for_clean ||
return 1
486 for i
in $
(seq 1 $objects)
488 rados
-p $poolname put obj
$i /dev
/null
491 local PG
=$
(get_pg
$poolname obj1
)
492 # Remember primary during the backfill
493 local primary
=$
(get_primary
$poolname obj1
)
495 ceph osd
set nobackfill
496 for o
in $
(get_osds
$poolname obj1
)
500 ceph osd
unset nobackfill
501 ceph tell osd.$
(get_primary
$poolname obj1
) debug kick_recovery_wq
0
504 wait_for_clean ||
return 1
506 local log
=$
(grep -l +backfilling
$dir/osd.
*.log
)
507 test -n "$log" ||
return 1
509 local misplaced
=$
(expr $objects \
* 3)
510 check
$PG $log 0 0 $misplaced $objects erasure ||
return 1
512 delete_pool
$poolname
513 kill_daemons
$dir ||
return 1
517 # [1,0,2] -> [4, 0, 2]
519 # active+remapped+backfilling
521 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
522 # 1.0 500 0 0 500 0 0 100 100 active+remapped+backfilling 2017-11-08 18:05:39.036420 24'500 27:742 [4,0,2] 4 [1,0,2] 1 0'0 2017-11-08 18:04:58.697315 0'0 2017-11-08 18:04:58.697315
523 function TEST_backfill_ec_prim_out
() {
526 run_mon
$dir a ||
return 1
527 run_mgr
$dir x ||
return 1
528 run_osd
$dir 0 ||
return 1
529 run_osd
$dir 1 ||
return 1
530 run_osd
$dir 2 ||
return 1
531 run_osd
$dir 3 ||
return 1
532 run_osd
$dir 4 ||
return 1
534 ceph osd erasure-code-profile
set myprofile plugin
=jerasure technique
=reed_sol_van k
=2 m
=1 crush-failure-domain
=osd
535 create_pool
$poolname 1 1 erasure myprofile
537 wait_for_clean ||
return 1
539 for i
in $
(seq 1 $objects)
541 rados
-p $poolname put obj
$i /dev
/null
544 local PG
=$
(get_pg
$poolname obj1
)
545 # Remember primary during the backfill
546 local primary
=$
(get_primary
$poolname obj1
)
548 ceph osd
set nobackfill
549 ceph osd out osd.
$primary
550 ceph osd
unset nobackfill
551 ceph tell osd.$
(get_primary
$poolname obj1
) debug kick_recovery_wq
0
554 wait_for_clean ||
return 1
556 local log
=$
(grep -l +backfilling
$dir/osd.
*.log
)
557 test -n "$log" ||
return 1
559 local misplaced
=$
(expr $objects \
* 3)
560 check
$PG $log 0 0 $objects 0 erasure ||
return 1
562 delete_pool
$poolname
563 kill_daemons
$dir ||
return 1
568 # misplaced 1000 -> 0
570 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
571 # 1.0 500 0 500 1000 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-11-06 14:02:29.439105 24'500 29:1020 [4,3,5] 4 [1,NONE,2] 1 0'0 2017-11-06 14:01:46.509963 0'0 2017-11-06 14:01:46.509963
572 function TEST_backfill_ec_down_all_out
() {
575 run_mon
$dir a ||
return 1
576 run_mgr
$dir x ||
return 1
577 run_osd
$dir 0 ||
return 1
578 run_osd
$dir 1 ||
return 1
579 run_osd
$dir 2 ||
return 1
580 run_osd
$dir 3 ||
return 1
581 run_osd
$dir 4 ||
return 1
582 run_osd
$dir 5 ||
return 1
584 ceph osd erasure-code-profile
set myprofile plugin
=jerasure technique
=reed_sol_van k
=2 m
=1 crush-failure-domain
=osd
585 create_pool
$poolname 1 1 erasure myprofile
586 ceph osd pool
set $poolname min_size
2
588 wait_for_clean ||
return 1
590 for i
in $
(seq 1 $objects)
592 rados
-p $poolname put obj
$i /dev
/null
595 local PG
=$
(get_pg
$poolname obj1
)
596 # Remember primary during the backfill
597 local primary
=$
(get_primary
$poolname obj1
)
598 local otherosd
=$
(get_not_primary
$poolname obj1
)
599 local allosds
=$
(get_osds
$poolname obj1
)
601 ceph osd
set nobackfill
602 kill $
(cat $dir/osd.
${otherosd}.pid
)
603 ceph osd down osd.
${otherosd}
608 ceph osd
unset nobackfill
609 ceph tell osd.$
(get_primary
$poolname obj1
) debug kick_recovery_wq
0
613 # Wait for recovery to finish
614 # Can't use wait_for_clean() because state goes from active+undersized+degraded+remapped+backfilling
615 # to active+undersized+remapped
618 if test "$(ceph --format json pg dump pgs |
619 jq '[.[] | .state | select(. == "incomplete
")] | length')" -ne "0"
629 if ceph pg dump pgs |
grep ^
$PG |
grep -qv backfilling
635 echo "Timeout waiting for recovery to finish"
643 local log
=$
(grep -l +backfilling
$dir/osd.
*.log
)
644 test -n "$log" ||
return 1
646 local misplaced
=$
(expr $objects \
* 2)
647 check
$PG $log $objects 0 $misplaced 0 erasure ||
return 1
649 delete_pool
$poolname
650 kill_daemons
$dir ||
return 1
656 # active+backfilling+degraded
658 # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
659 # 1.0 500 0 500 0 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-11-06 13:57:25.412322 22'500 28:794 [1,3,2] 1 [1,NONE,2] 1 0'0 2017-11-06 13:54:58.033906 0'0 2017-11-06 13:54:58.033906
660 function TEST_backfill_ec_down_out
() {
663 run_mon
$dir a ||
return 1
664 run_mgr
$dir x ||
return 1
665 run_osd
$dir 0 ||
return 1
666 run_osd
$dir 1 ||
return 1
667 run_osd
$dir 2 ||
return 1
668 run_osd
$dir 3 ||
return 1
669 run_osd
$dir 4 ||
return 1
670 run_osd
$dir 5 ||
return 1
672 ceph osd erasure-code-profile
set myprofile plugin
=jerasure technique
=reed_sol_van k
=2 m
=1 crush-failure-domain
=osd
673 create_pool
$poolname 1 1 erasure myprofile
674 ceph osd pool
set $poolname min_size
2
676 wait_for_clean ||
return 1
678 for i
in $
(seq 1 $objects)
680 rados
-p $poolname put obj
$i /dev
/null
683 local PG
=$
(get_pg
$poolname obj1
)
684 # Remember primary during the backfill
685 local primary
=$
(get_primary
$poolname obj1
)
686 local otherosd
=$
(get_not_primary
$poolname obj1
)
688 ceph osd
set nobackfill
689 kill $
(cat $dir/osd.
${otherosd}.pid
)
690 ceph osd down osd.
${otherosd}
691 ceph osd out osd.
${otherosd}
692 ceph osd
unset nobackfill
693 ceph tell osd.$
(get_primary
$poolname obj1
) debug kick_recovery_wq
0
696 wait_for_clean ||
return 1
698 local log
=$
(grep -l +backfilling
$dir/osd.
*.log
)
699 test -n "$log" ||
return 1
701 local misplaced
=$
(expr $objects \
* 2)
702 check
$PG $log $objects 0 0 0 erasure ||
return 1
704 delete_pool
$poolname
705 kill_daemons
$dir ||
return 1
709 main osd-backfill-stats
"$@"
712 # compile-command: "make -j4 && ../qa/run-standalone.sh osd-backfill-stats.sh"