]>
Commit | Line | Data |
---|---|---|
b32b8144 FG |
1 | #!/usr/bin/env bash |
2 | # | |
3 | # Copyright (C) 2017 Red Hat <contact@redhat.com> | |
4 | # | |
5 | # Author: David Zafman <dzafman@redhat.com> | |
6 | # | |
7 | # This program is free software; you can redistribute it and/or modify | |
8 | # it under the terms of the GNU Library Public License as published by | |
9 | # the Free Software Foundation; either version 2, or (at your option) | |
10 | # any later version. | |
11 | # | |
12 | # This program is distributed in the hope that it will be useful, | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | # GNU Library Public License for more details. | |
16 | # | |
17 | ||
18 | source $CEPH_ROOT/qa/standalone/ceph-helpers.sh | |
19 | ||
20 | function run() { | |
21 | local dir=$1 | |
22 | shift | |
23 | ||
24 | # Fix port???? | |
25 | export CEPH_MON="127.0.0.1:7114" # git grep '\<7114\>' : there must be only one | |
26 | export CEPH_ARGS | |
27 | CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " | |
28 | CEPH_ARGS+="--mon-host=$CEPH_MON " | |
29 | CEPH_ARGS+="--osd_min_pg_log_entries=5 --osd_max_pg_log_entries=10 " | |
30 | export margin=10 | |
31 | export objects=200 | |
32 | export poolname=test | |
33 | ||
34 | local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} | |
35 | for func in $funcs ; do | |
36 | setup $dir || return 1 | |
37 | $func $dir || return 1 | |
38 | teardown $dir || return 1 | |
39 | done | |
40 | } | |
41 | ||
42 | function below_margin() { | |
43 | local -i check=$1 | |
44 | shift | |
45 | local -i target=$1 | |
46 | ||
47 | return $(( $check <= $target && $check >= $target - $margin ? 0 : 1 )) | |
48 | } | |
49 | ||
50 | function above_margin() { | |
51 | local -i check=$1 | |
52 | shift | |
53 | local -i target=$1 | |
54 | ||
55 | return $(( $check >= $target && $check <= $target + $margin ? 0 : 1 )) | |
56 | } | |
57 | ||
58 | function check() { | |
59 | local PG=$1 | |
60 | local log=$2 | |
61 | local degraded_start=$3 | |
62 | local degraded_end=$4 | |
63 | local misplaced_start=$5 | |
64 | local misplaced_end=$6 | |
65 | local type=$7 | |
66 | ||
67 | local addp=" " | |
68 | if [ "$type" = "erasure" ]; | |
69 | then | |
70 | addp="p" | |
71 | fi | |
72 | ||
73 | UPACT=$(grep "pg[[]${PG}.*backfilling.*_update_calc_stats " $log | tail -1 | sed "s/.*[)] \([[][^ p]*\).*$/\1/") | |
74 | ||
75 | # Check 3rd line at start because of false backfill starts | |
76 | FIRST=$(grep "pg[[]${PG}.*backfilling.*_update_calc_stats degraded " $log | grep -F " ${UPACT}${addp}" | head -1 | sed "s/.* \([0-9]*\)$/\1/") | |
77 | below_margin $FIRST $degraded_start || return 1 | |
78 | LAST=$(grep "pg[[]${PG}.*backfilling.*_update_calc_stats degraded " $log | tail -1 | sed "s/.* \([0-9]*\)$/\1/") | |
79 | above_margin $LAST $degraded_end || return 1 | |
80 | ||
81 | # Check 3rd line at start because of false backfill starts | |
82 | FIRST=$(grep "pg[[]${PG}.*backfilling.*_update_calc_stats misplaced " $log | grep -F " ${UPACT}${addp}" | head -1 | sed "s/.* \([0-9]*\)$/\1/") | |
83 | below_margin $FIRST $misplaced_start || return 1 | |
84 | LAST=$(grep "pg[[]${PG}.*backfilling.*_update_calc_stats misplaced " $log | tail -1 | sed "s/.* \([0-9]*\)$/\1/") | |
85 | above_margin $LAST $misplaced_end || return 1 | |
86 | } | |
87 | ||
88 | # [1] -> [1, 0, 2] | |
89 | # degraded 1000 -> 0 | |
90 | # state: active+undersized+degraded+remapped+backfilling | |
91 | ||
92 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
93 | # 1.0 500 0 1000 0 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:44:23.531466 22'500 26:617 [1,0,2] 1 [1] 1 0'0 2017-10-27 09:43:44.654882 0'0 2017-10-27 09:43:44.654882 | |
94 | function TEST_backfill_sizeup() { | |
95 | local dir=$1 | |
96 | ||
97 | run_mon $dir a || return 1 | |
98 | run_mgr $dir x || return 1 | |
99 | export CEPH_ARGS | |
100 | run_osd $dir 0 || return 1 | |
101 | run_osd $dir 1 || return 1 | |
102 | run_osd $dir 2 || return 1 | |
103 | run_osd $dir 3 || return 1 | |
104 | run_osd $dir 4 || return 1 | |
105 | run_osd $dir 5 || return 1 | |
106 | ||
107 | create_pool $poolname 1 1 | |
108 | ceph osd pool set $poolname size 1 | |
109 | ||
110 | wait_for_clean || return 1 | |
111 | ||
112 | for i in $(seq 1 $objects) | |
113 | do | |
114 | rados -p $poolname put obj$i /dev/null | |
115 | done | |
116 | ||
117 | ceph osd pool set $poolname size 3 | |
118 | sleep 15 | |
119 | ||
120 | wait_for_clean || return 1 | |
121 | ||
122 | local primary=$(get_primary $poolname obj1) | |
123 | local PG=$(get_pg $poolname obj1) | |
124 | ||
125 | local log=$(grep -l +backfilling $dir/osd.*.log) | |
126 | test -n "$log" || return 1 | |
127 | ||
128 | local degraded=$(expr $objects \* 2) | |
129 | check $PG $log $degraded 0 0 0 || return 1 | |
130 | ||
131 | delete_pool $poolname | |
132 | kill_daemons $dir || return 1 | |
133 | } | |
134 | ||
135 | ||
136 | ||
137 | # [1] -> [0, 2, 4] | |
138 | # degraded 1000 -> 0 | |
139 | # misplaced 500 -> 0 | |
140 | # state: active+undersized+degraded+remapped+backfilling | |
141 | ||
142 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
143 | # 1.0 500 0 1000 500 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:48:53.326849 22'500 26:603 [0,2,4] 0 [1] 1 0'0 2017-10-27 09:48:13.236253 0'0 2017-10-27 09:48:13.236253 | |
144 | function TEST_backfill_sizeup_out() { | |
145 | local dir=$1 | |
146 | ||
147 | run_mon $dir a || return 1 | |
148 | run_mgr $dir x || return 1 | |
149 | run_osd $dir 0 || return 1 | |
150 | run_osd $dir 1 || return 1 | |
151 | run_osd $dir 2 || return 1 | |
152 | run_osd $dir 3 || return 1 | |
153 | run_osd $dir 4 || return 1 | |
154 | run_osd $dir 5 || return 1 | |
155 | ||
156 | create_pool $poolname 1 1 | |
157 | ceph osd pool set $poolname size 1 | |
158 | ||
159 | wait_for_clean || return 1 | |
160 | ||
161 | for i in $(seq 1 $objects) | |
162 | do | |
163 | rados -p $poolname put obj$i /dev/null | |
164 | done | |
165 | ||
166 | local PG=$(get_pg $poolname obj1) | |
167 | # Remember primary during the backfill | |
168 | local primary=$(get_primary $poolname obj1) | |
169 | ||
170 | ceph osd out osd.$primary | |
171 | ceph osd pool set $poolname size 3 | |
172 | sleep 15 | |
173 | ||
174 | wait_for_clean || return 1 | |
175 | ||
176 | local log=$(grep -l +backfilling $dir/osd.*.log) | |
177 | test -n "$log" || return 1 | |
178 | local degraded=$(expr $objects \* 2) | |
179 | check $PG $log $degraded 0 $objects 0 || return 1 | |
180 | ||
181 | delete_pool $poolname | |
182 | kill_daemons $dir || return 1 | |
183 | } | |
184 | ||
185 | ||
186 | # [1 0] -> [1,2]/[1,0] | |
187 | # misplaced 500 -> 0 | |
188 | # state: active+remapped+backfilling | |
189 | ||
190 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
191 | # 1.0 500 0 0 500 0 0 100 100 active+remapped+backfilling 2017-10-27 09:51:18.800517 22'500 25:570 [1,2] 1 [1,0] 1 0'0 2017-10-27 09:50:40.441274 0'0 2017-10-27 09:50:40.441274 | |
192 | function TEST_backfill_out() { | |
193 | local dir=$1 | |
194 | ||
195 | run_mon $dir a || return 1 | |
196 | run_mgr $dir x || return 1 | |
197 | run_osd $dir 0 || return 1 | |
198 | run_osd $dir 1 || return 1 | |
199 | run_osd $dir 2 || return 1 | |
200 | run_osd $dir 3 || return 1 | |
201 | run_osd $dir 4 || return 1 | |
202 | run_osd $dir 5 || return 1 | |
203 | ||
204 | create_pool $poolname 1 1 | |
205 | ceph osd pool set $poolname size 2 | |
206 | sleep 5 | |
207 | ||
208 | wait_for_clean || return 1 | |
209 | ||
210 | for i in $(seq 1 $objects) | |
211 | do | |
212 | rados -p $poolname put obj$i /dev/null | |
213 | done | |
214 | ||
215 | local PG=$(get_pg $poolname obj1) | |
216 | # Remember primary during the backfill | |
217 | local primary=$(get_primary $poolname obj1) | |
218 | ||
219 | ceph osd out osd.$(get_not_primary $poolname obj1) | |
220 | sleep 15 | |
221 | ||
222 | wait_for_clean || return 1 | |
223 | ||
224 | local log=$(grep -l +backfilling $dir/osd.*.log) | |
225 | test -n "$log" || return 1 | |
226 | ||
227 | check $PG $log 0 0 $objects 0 || return 1 | |
228 | ||
229 | delete_pool $poolname | |
230 | kill_daemons $dir || return 1 | |
231 | } | |
232 | ||
233 | ||
234 | # [0, 1] -> [0, 2]/[0] | |
235 | # osd 1 down/out | |
236 | # degraded 500 -> 0 | |
237 | # state: active+undersized+degraded+remapped+backfilling | |
238 | ||
239 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
240 | # 1.0 500 0 500 0 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:53:24.051091 22'500 27:719 [0,2] 0 [0] 0 0'0 2017-10-27 09:52:43.188368 0'0 2017-10-27 09:52:43.188368 | |
241 | function TEST_backfill_down_out() { | |
242 | local dir=$1 | |
243 | ||
244 | run_mon $dir a || return 1 | |
245 | run_mgr $dir x || return 1 | |
246 | run_osd $dir 0 || return 1 | |
247 | run_osd $dir 1 || return 1 | |
248 | run_osd $dir 2 || return 1 | |
249 | run_osd $dir 3 || return 1 | |
250 | run_osd $dir 4 || return 1 | |
251 | run_osd $dir 5 || return 1 | |
252 | ||
253 | create_pool $poolname 1 1 | |
254 | ceph osd pool set $poolname size 2 | |
255 | sleep 5 | |
256 | ||
257 | wait_for_clean || return 1 | |
258 | ||
259 | for i in $(seq 1 $objects) | |
260 | do | |
261 | rados -p $poolname put obj$i /dev/null | |
262 | done | |
263 | ||
264 | local PG=$(get_pg $poolname obj1) | |
265 | # Remember primary during the backfill | |
266 | local primary=$(get_primary $poolname obj1) | |
267 | local otherosd=$(get_not_primary $poolname obj1) | |
268 | ||
269 | kill $(cat $dir/osd.${otherosd}.pid) | |
270 | ceph osd down osd.${otherosd} | |
271 | ceph osd out osd.${otherosd} | |
272 | sleep 15 | |
273 | ||
274 | wait_for_clean || return 1 | |
275 | ||
276 | local log=$(grep -l +backfilling $dir/osd.*.log) | |
277 | test -n "$log" || return 1 | |
278 | ||
279 | check $PG $log $objects 0 0 0 || return 1 | |
280 | ||
281 | delete_pool $poolname | |
282 | kill_daemons $dir || return 1 | |
283 | } | |
284 | ||
285 | ||
286 | # [1, 0] -> [2, 3, 4] | |
287 | # degraded 500 -> 0 | |
288 | # misplaced 1000 -> 0 | |
289 | # state: active+undersized+degraded+remapped+backfilling | |
290 | ||
291 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
292 | # 1.0 500 0 500 1000 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:55:50.375722 23'500 27:553 [2,4,3] 2 [1,0] 1 0'0 2017-10-27 09:55:10.230919 0'0 2017-10-27 09:55:10.230919 | |
293 | function TEST_backfill_out2() { | |
294 | local dir=$1 | |
295 | ||
296 | run_mon $dir a || return 1 | |
297 | run_mgr $dir x || return 1 | |
298 | run_osd $dir 0 || return 1 | |
299 | run_osd $dir 1 || return 1 | |
300 | run_osd $dir 2 || return 1 | |
301 | run_osd $dir 3 || return 1 | |
302 | run_osd $dir 4 || return 1 | |
303 | run_osd $dir 5 || return 1 | |
304 | ||
305 | create_pool $poolname 1 1 | |
306 | ceph osd pool set $poolname size 2 | |
307 | sleep 5 | |
308 | ||
309 | wait_for_clean || return 1 | |
310 | ||
311 | for i in $(seq 1 $objects) | |
312 | do | |
313 | rados -p $poolname put obj$i /dev/null | |
314 | done | |
315 | ||
316 | local PG=$(get_pg $poolname obj1) | |
317 | # Remember primary during the backfill | |
318 | local primary=$(get_primary $poolname obj1) | |
319 | local otherosd=$(get_not_primary $poolname obj1) | |
320 | ||
321 | ceph osd set nobackfill | |
322 | ceph osd pool set $poolname size 3 | |
323 | ceph osd out osd.${otherosd} | |
324 | ceph osd out osd.${primary} | |
325 | ceph osd unset nobackfill | |
326 | ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0 | |
327 | sleep 2 | |
328 | ||
329 | wait_for_clean || return 1 | |
330 | ||
331 | local log=$(grep -l +backfilling $dir/osd.*.log) | |
332 | test -n "$log" || return 1 | |
333 | local misplaced=$(expr $objects \* 2) | |
334 | ||
335 | check $PG $log $objects 0 $misplaced 0 || return 1 | |
336 | ||
337 | delete_pool $poolname | |
338 | kill_daemons $dir || return 1 | |
339 | } | |
340 | ||
341 | ||
342 | # [0,1] -> [2,4,3]/[0,1] | |
343 | # degraded 1000 -> 0 | |
344 | # misplaced 1000 -> 500 | |
345 | # state ends at active+clean+remapped [2,4,3]/[2,4,3,0] | |
346 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
347 | # 1.0 500 0 1000 1000 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-10-30 18:21:45.995149 19'500 23:1817 [2,4,3] 2 [0,1] 0 0'0 2017-10-30 18:21:05.109904 0'0 2017-10-30 18:21:05.109904 | |
348 | # ENDS: | |
349 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
350 | # 1.0 500 0 0 500 0 0 5 5 active+clean+remapped 2017-10-30 18:22:42.293730 19'500 25:2557 [2,4,3] 2 [2,4,3,0] 2 0'0 2017-10-30 18:21:05.109904 0'0 2017-10-30 18:21:05.109904 | |
351 | function TEST_backfill_sizeup4_allout() { | |
352 | local dir=$1 | |
353 | ||
354 | run_mon $dir a || return 1 | |
355 | run_mgr $dir x || return 1 | |
356 | run_osd $dir 0 || return 1 | |
357 | run_osd $dir 1 || return 1 | |
358 | run_osd $dir 2 || return 1 | |
359 | run_osd $dir 3 || return 1 | |
360 | run_osd $dir 4 || return 1 | |
361 | ||
362 | create_pool $poolname 1 1 | |
363 | ceph osd pool set $poolname size 2 | |
364 | ||
365 | wait_for_clean || return 1 | |
366 | ||
367 | for i in $(seq 1 $objects) | |
368 | do | |
369 | rados -p $poolname put obj$i /dev/null | |
370 | done | |
371 | ||
372 | local PG=$(get_pg $poolname obj1) | |
373 | # Remember primary during the backfill | |
374 | local primary=$(get_primary $poolname obj1) | |
375 | local otherosd=$(get_not_primary $poolname obj1) | |
376 | ||
377 | ceph osd set nobackfill | |
378 | ceph osd out osd.$otherosd | |
379 | ceph osd out osd.$primary | |
380 | ceph osd pool set $poolname size 4 | |
381 | ceph osd unset nobackfill | |
382 | ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0 | |
383 | sleep 2 | |
384 | ||
385 | wait_for_clean || return 1 | |
386 | ||
387 | local log=$(grep -l +backfilling $dir/osd.*.log) | |
388 | test -n "$log" || return 1 | |
389 | local misdeg=$(expr $objects \* 2) | |
390 | check $PG $log $misdeg 0 $misdeg $objects || return 1 | |
391 | ||
392 | delete_pool $poolname | |
393 | kill_daemons $dir || return 1 | |
394 | } | |
395 | ||
396 | ||
397 | # [1,2,0] -> [3]/[1,2] | |
398 | # misplaced 1000 -> 500 | |
399 | # state ends at active+clean+remapped [3]/[3,1] | |
400 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
401 | # 1.0 500 0 0 1000 0 0 100 100 active+remapped+backfilling 2017-11-28 19:13:56.092439 21'500 31:790 [3] 3 [1,2] 1 0'0 2017-11-28 19:13:28.698661 0'0 2017-11-28 19:13:28.698661 | |
402 | function TEST_backfill_remapped() { | |
403 | local dir=$1 | |
404 | ||
405 | run_mon $dir a || return 1 | |
406 | run_mgr $dir x || return 1 | |
407 | run_osd $dir 0 || return 1 | |
408 | run_osd $dir 1 || return 1 | |
409 | run_osd $dir 2 || return 1 | |
410 | run_osd $dir 3 || return 1 | |
411 | ||
412 | create_pool $poolname 1 1 | |
413 | ceph osd pool set $poolname size 3 | |
414 | sleep 5 | |
415 | ||
416 | wait_for_clean || return 1 | |
417 | ||
418 | for i in $(seq 1 $objects) | |
419 | do | |
420 | rados -p $poolname put obj$i /dev/null | |
421 | done | |
422 | ||
423 | local PG=$(get_pg $poolname obj1) | |
424 | # Remember primary during the backfill | |
425 | local primary=$(get_primary $poolname obj1) | |
426 | local otherosd=$(get_not_primary $poolname obj1) | |
427 | ||
428 | ceph osd set nobackfill | |
429 | ceph osd out osd.${otherosd} | |
430 | for i in $(get_osds $poolname obj1) | |
431 | do | |
432 | if [ $i = $primary -o $i = $otherosd ]; | |
433 | then | |
434 | continue | |
435 | fi | |
436 | ceph osd out osd.$i | |
437 | break | |
438 | done | |
439 | ceph osd out osd.${primary} | |
440 | ceph osd pool set $poolname size 2 | |
441 | sleep 2 | |
442 | ceph osd unset nobackfill | |
443 | ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0 | |
444 | sleep 2 | |
445 | ||
446 | wait_for_clean || return 1 | |
447 | ||
448 | local log=$(grep -l +backfilling $dir/osd.*.log) | |
449 | test -n "$log" || return 1 | |
450 | local misplaced=$(expr $objects \* 2) | |
451 | ||
452 | check $PG $log 0 0 $misplaced $objects || return 1 | |
453 | ||
454 | delete_pool $poolname | |
455 | kill_daemons $dir || return 1 | |
456 | } | |
457 | ||
458 | # [1,0,2] -> [4,3,NONE]/[1,0,2] | |
459 | # misplaced 1500 -> 500 | |
460 | # state ends at active+clean+remapped [4,3,NONE]/[4,3,2] | |
461 | ||
462 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
463 | # 1.0 500 0 0 1500 0 0 100 100 active+degraded+remapped+backfilling 2017-10-31 16:53:39.467126 19'500 23:615 [4,3,NONE] 4 [1,0,2] 1 0'0 2017-10-31 16:52:59.624429 0'0 2017-10-31 16:52:59.624429 | |
464 | ||
465 | ||
466 | # ENDS: | |
467 | ||
468 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
469 | # 1.0 500 0 0 500 0 0 5 5 active+clean+remapped 2017-10-31 16:48:34.414040 19'500 25:2049 [4,3,NONE] 4 [4,3,2] 4 0'0 2017-10-31 16:46:58.203440 0'0 2017-10-31 16:46:58.203440 | |
470 | function TEST_backfill_ec_all_out() { | |
471 | local dir=$1 | |
472 | ||
473 | run_mon $dir a || return 1 | |
474 | run_mgr $dir x || return 1 | |
475 | run_osd $dir 0 || return 1 | |
476 | run_osd $dir 1 || return 1 | |
477 | run_osd $dir 2 || return 1 | |
478 | run_osd $dir 3 || return 1 | |
479 | run_osd $dir 4 || return 1 | |
480 | ||
481 | ceph osd erasure-code-profile set myprofile plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd | |
482 | create_pool $poolname 1 1 erasure myprofile | |
483 | ||
484 | wait_for_clean || return 1 | |
485 | ||
486 | for i in $(seq 1 $objects) | |
487 | do | |
488 | rados -p $poolname put obj$i /dev/null | |
489 | done | |
490 | ||
491 | local PG=$(get_pg $poolname obj1) | |
492 | # Remember primary during the backfill | |
493 | local primary=$(get_primary $poolname obj1) | |
494 | ||
495 | ceph osd set nobackfill | |
496 | for o in $(get_osds $poolname obj1) | |
497 | do | |
498 | ceph osd out osd.$o | |
499 | done | |
500 | ceph osd unset nobackfill | |
501 | ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0 | |
502 | sleep 2 | |
503 | ||
504 | wait_for_clean || return 1 | |
505 | ||
506 | local log=$(grep -l +backfilling $dir/osd.*.log) | |
507 | test -n "$log" || return 1 | |
508 | ||
509 | local misplaced=$(expr $objects \* 3) | |
510 | check $PG $log 0 0 $misplaced $objects erasure || return 1 | |
511 | ||
512 | delete_pool $poolname | |
513 | kill_daemons $dir || return 1 | |
514 | } | |
515 | ||
516 | ||
517 | # [1,0,2] -> [4, 0, 2] | |
518 | # misplaced 500 -> 0 | |
519 | # active+remapped+backfilling | |
520 | # | |
521 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
522 | # 1.0 500 0 0 500 0 0 100 100 active+remapped+backfilling 2017-11-08 18:05:39.036420 24'500 27:742 [4,0,2] 4 [1,0,2] 1 0'0 2017-11-08 18:04:58.697315 0'0 2017-11-08 18:04:58.697315 | |
523 | function TEST_backfill_ec_prim_out() { | |
524 | local dir=$1 | |
525 | ||
526 | run_mon $dir a || return 1 | |
527 | run_mgr $dir x || return 1 | |
528 | run_osd $dir 0 || return 1 | |
529 | run_osd $dir 1 || return 1 | |
530 | run_osd $dir 2 || return 1 | |
531 | run_osd $dir 3 || return 1 | |
532 | run_osd $dir 4 || return 1 | |
533 | ||
534 | ceph osd erasure-code-profile set myprofile plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd | |
535 | create_pool $poolname 1 1 erasure myprofile | |
536 | ||
537 | wait_for_clean || return 1 | |
538 | ||
539 | for i in $(seq 1 $objects) | |
540 | do | |
541 | rados -p $poolname put obj$i /dev/null | |
542 | done | |
543 | ||
544 | local PG=$(get_pg $poolname obj1) | |
545 | # Remember primary during the backfill | |
546 | local primary=$(get_primary $poolname obj1) | |
547 | ||
548 | ceph osd set nobackfill | |
549 | ceph osd out osd.$primary | |
550 | ceph osd unset nobackfill | |
551 | ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0 | |
552 | sleep 2 | |
553 | ||
554 | wait_for_clean || return 1 | |
555 | ||
556 | local log=$(grep -l +backfilling $dir/osd.*.log) | |
557 | test -n "$log" || return 1 | |
558 | ||
559 | local misplaced=$(expr $objects \* 3) | |
560 | check $PG $log 0 0 $objects 0 erasure || return 1 | |
561 | ||
562 | delete_pool $poolname | |
563 | kill_daemons $dir || return 1 | |
564 | } | |
565 | ||
566 | # [1,0] -> [1,2] | |
567 | # degraded 500 -> 0 | |
568 | # misplaced 1000 -> 0 | |
569 | # | |
570 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
571 | # 1.0 500 0 500 1000 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-11-06 14:02:29.439105 24'500 29:1020 [4,3,5] 4 [1,NONE,2] 1 0'0 2017-11-06 14:01:46.509963 0'0 2017-11-06 14:01:46.509963 | |
572 | function TEST_backfill_ec_down_all_out() { | |
573 | local dir=$1 | |
574 | ||
575 | run_mon $dir a || return 1 | |
576 | run_mgr $dir x || return 1 | |
577 | run_osd $dir 0 || return 1 | |
578 | run_osd $dir 1 || return 1 | |
579 | run_osd $dir 2 || return 1 | |
580 | run_osd $dir 3 || return 1 | |
581 | run_osd $dir 4 || return 1 | |
582 | run_osd $dir 5 || return 1 | |
583 | ||
584 | ceph osd erasure-code-profile set myprofile plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd | |
585 | create_pool $poolname 1 1 erasure myprofile | |
586 | ceph osd pool set $poolname min_size 2 | |
587 | ||
588 | wait_for_clean || return 1 | |
589 | ||
590 | for i in $(seq 1 $objects) | |
591 | do | |
592 | rados -p $poolname put obj$i /dev/null | |
593 | done | |
594 | ||
595 | local PG=$(get_pg $poolname obj1) | |
596 | # Remember primary during the backfill | |
597 | local primary=$(get_primary $poolname obj1) | |
598 | local otherosd=$(get_not_primary $poolname obj1) | |
599 | local allosds=$(get_osds $poolname obj1) | |
600 | ||
601 | ceph osd set nobackfill | |
602 | kill $(cat $dir/osd.${otherosd}.pid) | |
603 | ceph osd down osd.${otherosd} | |
604 | for o in $allosds | |
605 | do | |
606 | ceph osd out osd.$o | |
607 | done | |
608 | ceph osd unset nobackfill | |
609 | ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0 | |
610 | sleep 2 | |
611 | flush_pg_stats | |
612 | ||
613 | # Wait for recovery to finish | |
614 | # Can't use wait_for_clean() because state goes from active+undersized+degraded+remapped+backfilling | |
615 | # to active+undersized+remapped | |
616 | while(true) | |
617 | do | |
618 | if test "$(ceph --format json pg dump pgs | | |
619 | jq '[.[] | .state | select(. == "incomplete")] | length')" -ne "0" | |
620 | then | |
621 | sleep 2 | |
622 | continue | |
623 | fi | |
624 | break | |
625 | done | |
626 | ceph pg dump pgs | |
627 | for i in $(seq 1 60) | |
628 | do | |
629 | if ceph pg dump pgs | grep ^$PG | grep -qv backfilling | |
630 | then | |
631 | break | |
632 | fi | |
633 | if [ $i = "60" ]; | |
634 | then | |
635 | echo "Timeout waiting for recovery to finish" | |
636 | return 1 | |
637 | fi | |
638 | sleep 1 | |
639 | done | |
640 | ||
641 | ceph pg dump pgs | |
642 | ||
643 | local log=$(grep -l +backfilling $dir/osd.*.log) | |
644 | test -n "$log" || return 1 | |
645 | ||
646 | local misplaced=$(expr $objects \* 2) | |
647 | check $PG $log $objects 0 $misplaced 0 erasure || return 1 | |
648 | ||
649 | delete_pool $poolname | |
650 | kill_daemons $dir || return 1 | |
651 | } | |
652 | ||
653 | ||
654 | # [1,0,2] -> [1,3,2] | |
655 | # degraded 500 -> 0 | |
656 | # active+backfilling+degraded | |
657 | # | |
658 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
659 | # 1.0 500 0 500 0 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-11-06 13:57:25.412322 22'500 28:794 [1,3,2] 1 [1,NONE,2] 1 0'0 2017-11-06 13:54:58.033906 0'0 2017-11-06 13:54:58.033906 | |
660 | function TEST_backfill_ec_down_out() { | |
661 | local dir=$1 | |
662 | ||
663 | run_mon $dir a || return 1 | |
664 | run_mgr $dir x || return 1 | |
665 | run_osd $dir 0 || return 1 | |
666 | run_osd $dir 1 || return 1 | |
667 | run_osd $dir 2 || return 1 | |
668 | run_osd $dir 3 || return 1 | |
669 | run_osd $dir 4 || return 1 | |
670 | run_osd $dir 5 || return 1 | |
671 | ||
672 | ceph osd erasure-code-profile set myprofile plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd | |
673 | create_pool $poolname 1 1 erasure myprofile | |
674 | ceph osd pool set $poolname min_size 2 | |
675 | ||
676 | wait_for_clean || return 1 | |
677 | ||
678 | for i in $(seq 1 $objects) | |
679 | do | |
680 | rados -p $poolname put obj$i /dev/null | |
681 | done | |
682 | ||
683 | local PG=$(get_pg $poolname obj1) | |
684 | # Remember primary during the backfill | |
685 | local primary=$(get_primary $poolname obj1) | |
686 | local otherosd=$(get_not_primary $poolname obj1) | |
687 | ||
688 | ceph osd set nobackfill | |
689 | kill $(cat $dir/osd.${otherosd}.pid) | |
690 | ceph osd down osd.${otherosd} | |
691 | ceph osd out osd.${otherosd} | |
692 | ceph osd unset nobackfill | |
693 | ceph tell osd.$(get_primary $poolname obj1) debug kick_recovery_wq 0 | |
694 | sleep 2 | |
695 | ||
696 | wait_for_clean || return 1 | |
697 | ||
698 | local log=$(grep -l +backfilling $dir/osd.*.log) | |
699 | test -n "$log" || return 1 | |
700 | ||
701 | local misplaced=$(expr $objects \* 2) | |
702 | check $PG $log $objects 0 0 0 erasure || return 1 | |
703 | ||
704 | delete_pool $poolname | |
705 | kill_daemons $dir || return 1 | |
706 | } | |
707 | ||
708 | ||
709 | main recout "$@" | |
710 | ||
711 | # Local Variables: | |
712 | # compile-command: "make -j4 && ../qa/run-standalone.sh osd-backfill-stats.sh" | |
713 | # End: |