]>
Commit | Line | Data |
---|---|---|
b32b8144 FG |
1 | #!/usr/bin/env bash |
2 | # | |
3 | # Copyright (C) 2017 Red Hat <contact@redhat.com> | |
4 | # | |
5 | # Author: David Zafman <dzafman@redhat.com> | |
6 | # | |
7 | # This program is free software; you can redistribute it and/or modify | |
8 | # it under the terms of the GNU Library Public License as published by | |
9 | # the Free Software Foundation; either version 2, or (at your option) | |
10 | # any later version. | |
11 | # | |
12 | # This program is distributed in the hope that it will be useful, | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | # GNU Library Public License for more details. | |
16 | # | |
17 | ||
18 | source $CEPH_ROOT/qa/standalone/ceph-helpers.sh | |
19 | ||
20 | function run() { | |
21 | local dir=$1 | |
22 | shift | |
23 | ||
24 | # Fix port???? | |
25 | export CEPH_MON="127.0.0.1:7114" # git grep '\<7114\>' : there must be only one | |
26 | export CEPH_ARGS | |
27 | CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " | |
28 | CEPH_ARGS+="--mon-host=$CEPH_MON " | |
29 | CEPH_ARGS+="--osd_min_pg_log_entries=5 --osd_max_pg_log_entries=10 " | |
30 | export margin=10 | |
31 | export objects=200 | |
32 | export poolname=test | |
33 | ||
34 | local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} | |
35 | for func in $funcs ; do | |
36 | setup $dir || return 1 | |
37 | $func $dir || return 1 | |
38 | teardown $dir || return 1 | |
39 | done | |
40 | } | |
41 | ||
42 | function below_margin() { | |
43 | local -i check=$1 | |
44 | shift | |
45 | local -i target=$1 | |
46 | ||
47 | return $(( $check <= $target && $check >= $target - $margin ? 0 : 1 )) | |
48 | } | |
49 | ||
50 | function above_margin() { | |
51 | local -i check=$1 | |
52 | shift | |
53 | local -i target=$1 | |
54 | ||
55 | return $(( $check >= $target && $check <= $target + $margin ? 0 : 1 )) | |
56 | } | |
57 | ||
91327a77 AA |
58 | FIND_UPACT='grep "pg[[]${PG}.*backfilling.*_update_calc_stats " $log | tail -1 | sed "s/.*[)] \([[][^ p]*\).*$/\1/"' |
59 | FIND_FIRST='grep "pg[[]${PG}.*backfilling.*_update_calc_stats $which " $log | grep -F " ${UPACT}${addp}" | grep -v est | head -1 | sed "s/.* \([0-9]*\)$/\1/"' | |
60 | FIND_LAST='grep "pg[[]${PG}.*backfilling.*_update_calc_stats $which " $log | tail -1 | sed "s/.* \([0-9]*\)$/\1/"' | |
61 | ||
b32b8144 | 62 | function check() { |
91327a77 AA |
63 | local dir=$1 |
64 | local PG=$2 | |
65 | local primary=$3 | |
66 | local type=$4 | |
67 | local degraded_start=$5 | |
68 | local degraded_end=$6 | |
69 | local misplaced_start=$7 | |
70 | local misplaced_end=$8 | |
71 | local primary_start=${9:-} | |
72 | local primary_end=${10:-} | |
11fdf7f2 | 73 | local check_setup=${11:-true} |
91327a77 | 74 | |
11fdf7f2 TL |
75 | local log=$(grep -l +backfilling $dir/osd.$primary.log) |
76 | if [ $check_setup = "true" ]; | |
77 | then | |
78 | local alllogs=$(grep -l +backfilling $dir/osd.*.log) | |
79 | if [ "$(echo "$alllogs" | wc -w)" != "1" ]; | |
80 | then | |
81 | echo "Test setup failure, a single OSD should have performed backfill" | |
82 | return 1 | |
83 | fi | |
84 | fi | |
b32b8144 FG |
85 | |
86 | local addp=" " | |
87 | if [ "$type" = "erasure" ]; | |
88 | then | |
89 | addp="p" | |
90 | fi | |
91 | ||
91327a77 AA |
92 | UPACT=$(eval $FIND_UPACT) |
93 | [ -n "$UPACT" ] || return 1 | |
b32b8144 | 94 | |
91327a77 AA |
95 | # Check 3rd line at start because of false recovery starts |
96 | local which="degraded" | |
97 | FIRST=$(eval $FIND_FIRST) | |
98 | [ -n "$FIRST" ] || return 1 | |
b32b8144 | 99 | below_margin $FIRST $degraded_start || return 1 |
91327a77 AA |
100 | LAST=$(eval $FIND_LAST) |
101 | [ -n "$LAST" ] || return 1 | |
b32b8144 FG |
102 | above_margin $LAST $degraded_end || return 1 |
103 | ||
91327a77 AA |
104 | # Check 3rd line at start because of false recovery starts |
105 | which="misplaced" | |
106 | FIRST=$(eval $FIND_FIRST) | |
107 | [ -n "$FIRST" ] || return 1 | |
b32b8144 | 108 | below_margin $FIRST $misplaced_start || return 1 |
91327a77 AA |
109 | LAST=$(eval $FIND_LAST) |
110 | [ -n "$LAST" ] || return 1 | |
b32b8144 | 111 | above_margin $LAST $misplaced_end || return 1 |
91327a77 AA |
112 | |
113 | # This is the value of set into MISSING_ON_PRIMARY | |
114 | if [ -n "$primary_start" ]; | |
115 | then | |
116 | which="shard $primary" | |
117 | FIRST=$(eval $FIND_FIRST) | |
118 | [ -n "$FIRST" ] || return 1 | |
119 | below_margin $FIRST $primary_start || return 1 | |
120 | LAST=$(eval $FIND_LAST) | |
121 | [ -n "$LAST" ] || return 1 | |
122 | above_margin $LAST $primary_end || return 1 | |
123 | fi | |
b32b8144 FG |
124 | } |
125 | ||
126 | # [1] -> [1, 0, 2] | |
127 | # degraded 1000 -> 0 | |
128 | # state: active+undersized+degraded+remapped+backfilling | |
129 | ||
130 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
131 | # 1.0 500 0 1000 0 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:44:23.531466 22'500 26:617 [1,0,2] 1 [1] 1 0'0 2017-10-27 09:43:44.654882 0'0 2017-10-27 09:43:44.654882 | |
132 | function TEST_backfill_sizeup() { | |
133 | local dir=$1 | |
134 | ||
135 | run_mon $dir a || return 1 | |
136 | run_mgr $dir x || return 1 | |
137 | export CEPH_ARGS | |
138 | run_osd $dir 0 || return 1 | |
139 | run_osd $dir 1 || return 1 | |
140 | run_osd $dir 2 || return 1 | |
141 | run_osd $dir 3 || return 1 | |
142 | run_osd $dir 4 || return 1 | |
143 | run_osd $dir 5 || return 1 | |
144 | ||
145 | create_pool $poolname 1 1 | |
146 | ceph osd pool set $poolname size 1 | |
147 | ||
148 | wait_for_clean || return 1 | |
149 | ||
150 | for i in $(seq 1 $objects) | |
151 | do | |
152 | rados -p $poolname put obj$i /dev/null | |
153 | done | |
154 | ||
155 | ceph osd pool set $poolname size 3 | |
156 | sleep 15 | |
157 | ||
158 | wait_for_clean || return 1 | |
159 | ||
160 | local primary=$(get_primary $poolname obj1) | |
161 | local PG=$(get_pg $poolname obj1) | |
162 | ||
b32b8144 | 163 | local degraded=$(expr $objects \* 2) |
91327a77 | 164 | check $dir $PG $primary replicated $degraded 0 0 0 || return 1 |
b32b8144 FG |
165 | |
166 | delete_pool $poolname | |
167 | kill_daemons $dir || return 1 | |
168 | } | |
169 | ||
170 | ||
171 | ||
172 | # [1] -> [0, 2, 4] | |
173 | # degraded 1000 -> 0 | |
174 | # misplaced 500 -> 0 | |
175 | # state: active+undersized+degraded+remapped+backfilling | |
176 | ||
177 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
178 | # 1.0 500 0 1000 500 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:48:53.326849 22'500 26:603 [0,2,4] 0 [1] 1 0'0 2017-10-27 09:48:13.236253 0'0 2017-10-27 09:48:13.236253 | |
179 | function TEST_backfill_sizeup_out() { | |
180 | local dir=$1 | |
181 | ||
182 | run_mon $dir a || return 1 | |
183 | run_mgr $dir x || return 1 | |
184 | run_osd $dir 0 || return 1 | |
185 | run_osd $dir 1 || return 1 | |
186 | run_osd $dir 2 || return 1 | |
187 | run_osd $dir 3 || return 1 | |
188 | run_osd $dir 4 || return 1 | |
189 | run_osd $dir 5 || return 1 | |
190 | ||
191 | create_pool $poolname 1 1 | |
192 | ceph osd pool set $poolname size 1 | |
193 | ||
194 | wait_for_clean || return 1 | |
195 | ||
196 | for i in $(seq 1 $objects) | |
197 | do | |
198 | rados -p $poolname put obj$i /dev/null | |
199 | done | |
200 | ||
201 | local PG=$(get_pg $poolname obj1) | |
202 | # Remember primary during the backfill | |
203 | local primary=$(get_primary $poolname obj1) | |
204 | ||
205 | ceph osd out osd.$primary | |
206 | ceph osd pool set $poolname size 3 | |
207 | sleep 15 | |
208 | ||
209 | wait_for_clean || return 1 | |
210 | ||
b32b8144 | 211 | local degraded=$(expr $objects \* 2) |
91327a77 | 212 | check $dir $PG $primary replicated $degraded 0 $objects 0 || return 1 |
b32b8144 FG |
213 | |
214 | delete_pool $poolname | |
215 | kill_daemons $dir || return 1 | |
216 | } | |
217 | ||
218 | ||
219 | # [1 0] -> [1,2]/[1,0] | |
220 | # misplaced 500 -> 0 | |
221 | # state: active+remapped+backfilling | |
222 | ||
223 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
224 | # 1.0 500 0 0 500 0 0 100 100 active+remapped+backfilling 2017-10-27 09:51:18.800517 22'500 25:570 [1,2] 1 [1,0] 1 0'0 2017-10-27 09:50:40.441274 0'0 2017-10-27 09:50:40.441274 | |
225 | function TEST_backfill_out() { | |
226 | local dir=$1 | |
227 | ||
228 | run_mon $dir a || return 1 | |
229 | run_mgr $dir x || return 1 | |
230 | run_osd $dir 0 || return 1 | |
231 | run_osd $dir 1 || return 1 | |
232 | run_osd $dir 2 || return 1 | |
233 | run_osd $dir 3 || return 1 | |
234 | run_osd $dir 4 || return 1 | |
235 | run_osd $dir 5 || return 1 | |
236 | ||
237 | create_pool $poolname 1 1 | |
238 | ceph osd pool set $poolname size 2 | |
239 | sleep 5 | |
240 | ||
241 | wait_for_clean || return 1 | |
242 | ||
243 | for i in $(seq 1 $objects) | |
244 | do | |
245 | rados -p $poolname put obj$i /dev/null | |
246 | done | |
247 | ||
248 | local PG=$(get_pg $poolname obj1) | |
249 | # Remember primary during the backfill | |
250 | local primary=$(get_primary $poolname obj1) | |
251 | ||
252 | ceph osd out osd.$(get_not_primary $poolname obj1) | |
253 | sleep 15 | |
254 | ||
255 | wait_for_clean || return 1 | |
256 | ||
91327a77 | 257 | check $dir $PG $primary replicated 0 0 $objects 0 || return 1 |
b32b8144 FG |
258 | |
259 | delete_pool $poolname | |
260 | kill_daemons $dir || return 1 | |
261 | } | |
262 | ||
263 | ||
264 | # [0, 1] -> [0, 2]/[0] | |
265 | # osd 1 down/out | |
266 | # degraded 500 -> 0 | |
267 | # state: active+undersized+degraded+remapped+backfilling | |
268 | ||
269 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
270 | # 1.0 500 0 500 0 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:53:24.051091 22'500 27:719 [0,2] 0 [0] 0 0'0 2017-10-27 09:52:43.188368 0'0 2017-10-27 09:52:43.188368 | |
271 | function TEST_backfill_down_out() { | |
272 | local dir=$1 | |
273 | ||
274 | run_mon $dir a || return 1 | |
275 | run_mgr $dir x || return 1 | |
276 | run_osd $dir 0 || return 1 | |
277 | run_osd $dir 1 || return 1 | |
278 | run_osd $dir 2 || return 1 | |
279 | run_osd $dir 3 || return 1 | |
280 | run_osd $dir 4 || return 1 | |
281 | run_osd $dir 5 || return 1 | |
282 | ||
283 | create_pool $poolname 1 1 | |
284 | ceph osd pool set $poolname size 2 | |
285 | sleep 5 | |
286 | ||
287 | wait_for_clean || return 1 | |
288 | ||
289 | for i in $(seq 1 $objects) | |
290 | do | |
291 | rados -p $poolname put obj$i /dev/null | |
292 | done | |
293 | ||
294 | local PG=$(get_pg $poolname obj1) | |
295 | # Remember primary during the backfill | |
296 | local primary=$(get_primary $poolname obj1) | |
297 | local otherosd=$(get_not_primary $poolname obj1) | |
298 | ||
299 | kill $(cat $dir/osd.${otherosd}.pid) | |
300 | ceph osd down osd.${otherosd} | |
301 | ceph osd out osd.${otherosd} | |
302 | sleep 15 | |
303 | ||
304 | wait_for_clean || return 1 | |
305 | ||
91327a77 | 306 | check $dir $PG $primary replicated $objects 0 0 0 || return 1 |
b32b8144 FG |
307 | |
308 | delete_pool $poolname | |
309 | kill_daemons $dir || return 1 | |
310 | } | |
311 | ||
312 | ||
313 | # [1, 0] -> [2, 3, 4] | |
314 | # degraded 500 -> 0 | |
315 | # misplaced 1000 -> 0 | |
316 | # state: active+undersized+degraded+remapped+backfilling | |
317 | ||
318 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
319 | # 1.0 500 0 500 1000 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-10-27 09:55:50.375722 23'500 27:553 [2,4,3] 2 [1,0] 1 0'0 2017-10-27 09:55:10.230919 0'0 2017-10-27 09:55:10.230919 | |
320 | function TEST_backfill_out2() { | |
321 | local dir=$1 | |
322 | ||
323 | run_mon $dir a || return 1 | |
324 | run_mgr $dir x || return 1 | |
325 | run_osd $dir 0 || return 1 | |
326 | run_osd $dir 1 || return 1 | |
327 | run_osd $dir 2 || return 1 | |
328 | run_osd $dir 3 || return 1 | |
329 | run_osd $dir 4 || return 1 | |
330 | run_osd $dir 5 || return 1 | |
331 | ||
332 | create_pool $poolname 1 1 | |
333 | ceph osd pool set $poolname size 2 | |
334 | sleep 5 | |
335 | ||
336 | wait_for_clean || return 1 | |
337 | ||
338 | for i in $(seq 1 $objects) | |
339 | do | |
340 | rados -p $poolname put obj$i /dev/null | |
341 | done | |
342 | ||
343 | local PG=$(get_pg $poolname obj1) | |
344 | # Remember primary during the backfill | |
345 | local primary=$(get_primary $poolname obj1) | |
346 | local otherosd=$(get_not_primary $poolname obj1) | |
347 | ||
348 | ceph osd set nobackfill | |
349 | ceph osd pool set $poolname size 3 | |
350 | ceph osd out osd.${otherosd} | |
351 | ceph osd out osd.${primary} | |
11fdf7f2 TL |
352 | # Primary might change before backfill starts |
353 | sleep 2 | |
354 | primary=$(get_primary $poolname obj1) | |
b32b8144 | 355 | ceph osd unset nobackfill |
11fdf7f2 | 356 | ceph tell osd.$primary debug kick_recovery_wq 0 |
b32b8144 FG |
357 | sleep 2 |
358 | ||
359 | wait_for_clean || return 1 | |
360 | ||
b32b8144 FG |
361 | local misplaced=$(expr $objects \* 2) |
362 | ||
91327a77 | 363 | check $dir $PG $primary replicated $objects 0 $misplaced 0 || return 1 |
b32b8144 FG |
364 | |
365 | delete_pool $poolname | |
366 | kill_daemons $dir || return 1 | |
367 | } | |
368 | ||
369 | ||
370 | # [0,1] -> [2,4,3]/[0,1] | |
371 | # degraded 1000 -> 0 | |
372 | # misplaced 1000 -> 500 | |
373 | # state ends at active+clean+remapped [2,4,3]/[2,4,3,0] | |
374 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
375 | # 1.0 500 0 1000 1000 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-10-30 18:21:45.995149 19'500 23:1817 [2,4,3] 2 [0,1] 0 0'0 2017-10-30 18:21:05.109904 0'0 2017-10-30 18:21:05.109904 | |
376 | # ENDS: | |
377 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
378 | # 1.0 500 0 0 500 0 0 5 5 active+clean+remapped 2017-10-30 18:22:42.293730 19'500 25:2557 [2,4,3] 2 [2,4,3,0] 2 0'0 2017-10-30 18:21:05.109904 0'0 2017-10-30 18:21:05.109904 | |
379 | function TEST_backfill_sizeup4_allout() { | |
380 | local dir=$1 | |
381 | ||
382 | run_mon $dir a || return 1 | |
383 | run_mgr $dir x || return 1 | |
384 | run_osd $dir 0 || return 1 | |
385 | run_osd $dir 1 || return 1 | |
386 | run_osd $dir 2 || return 1 | |
387 | run_osd $dir 3 || return 1 | |
388 | run_osd $dir 4 || return 1 | |
389 | ||
390 | create_pool $poolname 1 1 | |
391 | ceph osd pool set $poolname size 2 | |
392 | ||
393 | wait_for_clean || return 1 | |
394 | ||
395 | for i in $(seq 1 $objects) | |
396 | do | |
397 | rados -p $poolname put obj$i /dev/null | |
398 | done | |
399 | ||
400 | local PG=$(get_pg $poolname obj1) | |
401 | # Remember primary during the backfill | |
402 | local primary=$(get_primary $poolname obj1) | |
403 | local otherosd=$(get_not_primary $poolname obj1) | |
404 | ||
405 | ceph osd set nobackfill | |
406 | ceph osd out osd.$otherosd | |
407 | ceph osd out osd.$primary | |
408 | ceph osd pool set $poolname size 4 | |
11fdf7f2 TL |
409 | # Primary might change before backfill starts |
410 | sleep 2 | |
411 | primary=$(get_primary $poolname obj1) | |
b32b8144 | 412 | ceph osd unset nobackfill |
11fdf7f2 | 413 | ceph tell osd.$primary debug kick_recovery_wq 0 |
b32b8144 FG |
414 | sleep 2 |
415 | ||
416 | wait_for_clean || return 1 | |
417 | ||
b32b8144 | 418 | local misdeg=$(expr $objects \* 2) |
91327a77 | 419 | check $dir $PG $primary replicated $misdeg 0 $misdeg $objects || return 1 |
b32b8144 FG |
420 | |
421 | delete_pool $poolname | |
422 | kill_daemons $dir || return 1 | |
423 | } | |
424 | ||
425 | ||
426 | # [1,2,0] -> [3]/[1,2] | |
427 | # misplaced 1000 -> 500 | |
428 | # state ends at active+clean+remapped [3]/[3,1] | |
429 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
430 | # 1.0 500 0 0 1000 0 0 100 100 active+remapped+backfilling 2017-11-28 19:13:56.092439 21'500 31:790 [3] 3 [1,2] 1 0'0 2017-11-28 19:13:28.698661 0'0 2017-11-28 19:13:28.698661 | |
431 | function TEST_backfill_remapped() { | |
432 | local dir=$1 | |
433 | ||
434 | run_mon $dir a || return 1 | |
435 | run_mgr $dir x || return 1 | |
436 | run_osd $dir 0 || return 1 | |
437 | run_osd $dir 1 || return 1 | |
438 | run_osd $dir 2 || return 1 | |
439 | run_osd $dir 3 || return 1 | |
440 | ||
441 | create_pool $poolname 1 1 | |
442 | ceph osd pool set $poolname size 3 | |
443 | sleep 5 | |
444 | ||
445 | wait_for_clean || return 1 | |
446 | ||
447 | for i in $(seq 1 $objects) | |
448 | do | |
449 | rados -p $poolname put obj$i /dev/null | |
450 | done | |
451 | ||
452 | local PG=$(get_pg $poolname obj1) | |
453 | # Remember primary during the backfill | |
454 | local primary=$(get_primary $poolname obj1) | |
455 | local otherosd=$(get_not_primary $poolname obj1) | |
456 | ||
457 | ceph osd set nobackfill | |
458 | ceph osd out osd.${otherosd} | |
459 | for i in $(get_osds $poolname obj1) | |
460 | do | |
461 | if [ $i = $primary -o $i = $otherosd ]; | |
462 | then | |
463 | continue | |
464 | fi | |
465 | ceph osd out osd.$i | |
466 | break | |
467 | done | |
468 | ceph osd out osd.${primary} | |
469 | ceph osd pool set $poolname size 2 | |
470 | sleep 2 | |
11fdf7f2 TL |
471 | |
472 | # primary may change due to invalidating the old pg_temp, which was [1,2,0], | |
473 | # but up_primary (3) chooses [0,1] for acting. | |
474 | primary=$(get_primary $poolname obj1) | |
475 | ||
b32b8144 | 476 | ceph osd unset nobackfill |
11fdf7f2 TL |
477 | ceph tell osd.$primary debug kick_recovery_wq 0 |
478 | ||
b32b8144 FG |
479 | sleep 2 |
480 | ||
481 | wait_for_clean || return 1 | |
482 | ||
b32b8144 FG |
483 | local misplaced=$(expr $objects \* 2) |
484 | ||
11fdf7f2 | 485 | check $dir $PG $primary replicated 0 0 $misplaced $objects "" "" false || return 1 |
b32b8144 FG |
486 | |
487 | delete_pool $poolname | |
488 | kill_daemons $dir || return 1 | |
489 | } | |
490 | ||
491 | # [1,0,2] -> [4,3,NONE]/[1,0,2] | |
492 | # misplaced 1500 -> 500 | |
493 | # state ends at active+clean+remapped [4,3,NONE]/[4,3,2] | |
494 | ||
495 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
496 | # 1.0 500 0 0 1500 0 0 100 100 active+degraded+remapped+backfilling 2017-10-31 16:53:39.467126 19'500 23:615 [4,3,NONE] 4 [1,0,2] 1 0'0 2017-10-31 16:52:59.624429 0'0 2017-10-31 16:52:59.624429 | |
497 | ||
498 | ||
499 | # ENDS: | |
500 | ||
501 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
502 | # 1.0 500 0 0 500 0 0 5 5 active+clean+remapped 2017-10-31 16:48:34.414040 19'500 25:2049 [4,3,NONE] 4 [4,3,2] 4 0'0 2017-10-31 16:46:58.203440 0'0 2017-10-31 16:46:58.203440 | |
503 | function TEST_backfill_ec_all_out() { | |
504 | local dir=$1 | |
505 | ||
506 | run_mon $dir a || return 1 | |
507 | run_mgr $dir x || return 1 | |
508 | run_osd $dir 0 || return 1 | |
509 | run_osd $dir 1 || return 1 | |
510 | run_osd $dir 2 || return 1 | |
511 | run_osd $dir 3 || return 1 | |
512 | run_osd $dir 4 || return 1 | |
513 | ||
514 | ceph osd erasure-code-profile set myprofile plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd | |
515 | create_pool $poolname 1 1 erasure myprofile | |
516 | ||
517 | wait_for_clean || return 1 | |
518 | ||
519 | for i in $(seq 1 $objects) | |
520 | do | |
521 | rados -p $poolname put obj$i /dev/null | |
522 | done | |
523 | ||
524 | local PG=$(get_pg $poolname obj1) | |
525 | # Remember primary during the backfill | |
526 | local primary=$(get_primary $poolname obj1) | |
527 | ||
528 | ceph osd set nobackfill | |
529 | for o in $(get_osds $poolname obj1) | |
530 | do | |
531 | ceph osd out osd.$o | |
532 | done | |
11fdf7f2 TL |
533 | # Primary might change before backfill starts |
534 | sleep 2 | |
535 | primary=$(get_primary $poolname obj1) | |
b32b8144 | 536 | ceph osd unset nobackfill |
11fdf7f2 | 537 | ceph tell osd.$primary debug kick_recovery_wq 0 |
b32b8144 FG |
538 | sleep 2 |
539 | ||
540 | wait_for_clean || return 1 | |
541 | ||
b32b8144 | 542 | local misplaced=$(expr $objects \* 3) |
91327a77 | 543 | check $dir $PG $primary erasure 0 0 $misplaced $objects || return 1 |
b32b8144 FG |
544 | |
545 | delete_pool $poolname | |
546 | kill_daemons $dir || return 1 | |
547 | } | |
548 | ||
549 | ||
550 | # [1,0,2] -> [4, 0, 2] | |
551 | # misplaced 500 -> 0 | |
552 | # active+remapped+backfilling | |
553 | # | |
554 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
555 | # 1.0 500 0 0 500 0 0 100 100 active+remapped+backfilling 2017-11-08 18:05:39.036420 24'500 27:742 [4,0,2] 4 [1,0,2] 1 0'0 2017-11-08 18:04:58.697315 0'0 2017-11-08 18:04:58.697315 | |
556 | function TEST_backfill_ec_prim_out() { | |
557 | local dir=$1 | |
558 | ||
559 | run_mon $dir a || return 1 | |
560 | run_mgr $dir x || return 1 | |
561 | run_osd $dir 0 || return 1 | |
562 | run_osd $dir 1 || return 1 | |
563 | run_osd $dir 2 || return 1 | |
564 | run_osd $dir 3 || return 1 | |
565 | run_osd $dir 4 || return 1 | |
566 | ||
567 | ceph osd erasure-code-profile set myprofile plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd | |
568 | create_pool $poolname 1 1 erasure myprofile | |
569 | ||
570 | wait_for_clean || return 1 | |
571 | ||
572 | for i in $(seq 1 $objects) | |
573 | do | |
574 | rados -p $poolname put obj$i /dev/null | |
575 | done | |
576 | ||
577 | local PG=$(get_pg $poolname obj1) | |
578 | # Remember primary during the backfill | |
579 | local primary=$(get_primary $poolname obj1) | |
580 | ||
581 | ceph osd set nobackfill | |
582 | ceph osd out osd.$primary | |
11fdf7f2 TL |
583 | # Primary might change before backfill starts |
584 | sleep 2 | |
585 | primary=$(get_primary $poolname obj1) | |
b32b8144 | 586 | ceph osd unset nobackfill |
11fdf7f2 | 587 | ceph tell osd.$primary debug kick_recovery_wq 0 |
b32b8144 FG |
588 | sleep 2 |
589 | ||
590 | wait_for_clean || return 1 | |
591 | ||
b32b8144 | 592 | local misplaced=$(expr $objects \* 3) |
91327a77 | 593 | check $dir $PG $primary erasure 0 0 $objects 0 || return 1 |
b32b8144 FG |
594 | |
595 | delete_pool $poolname | |
596 | kill_daemons $dir || return 1 | |
597 | } | |
598 | ||
599 | # [1,0] -> [1,2] | |
600 | # degraded 500 -> 0 | |
601 | # misplaced 1000 -> 0 | |
602 | # | |
603 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
604 | # 1.0 500 0 500 1000 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-11-06 14:02:29.439105 24'500 29:1020 [4,3,5] 4 [1,NONE,2] 1 0'0 2017-11-06 14:01:46.509963 0'0 2017-11-06 14:01:46.509963 | |
605 | function TEST_backfill_ec_down_all_out() { | |
606 | local dir=$1 | |
607 | ||
608 | run_mon $dir a || return 1 | |
609 | run_mgr $dir x || return 1 | |
610 | run_osd $dir 0 || return 1 | |
611 | run_osd $dir 1 || return 1 | |
612 | run_osd $dir 2 || return 1 | |
613 | run_osd $dir 3 || return 1 | |
614 | run_osd $dir 4 || return 1 | |
615 | run_osd $dir 5 || return 1 | |
616 | ||
617 | ceph osd erasure-code-profile set myprofile plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd | |
618 | create_pool $poolname 1 1 erasure myprofile | |
619 | ceph osd pool set $poolname min_size 2 | |
620 | ||
621 | wait_for_clean || return 1 | |
622 | ||
623 | for i in $(seq 1 $objects) | |
624 | do | |
625 | rados -p $poolname put obj$i /dev/null | |
626 | done | |
627 | ||
628 | local PG=$(get_pg $poolname obj1) | |
629 | # Remember primary during the backfill | |
630 | local primary=$(get_primary $poolname obj1) | |
631 | local otherosd=$(get_not_primary $poolname obj1) | |
632 | local allosds=$(get_osds $poolname obj1) | |
633 | ||
634 | ceph osd set nobackfill | |
635 | kill $(cat $dir/osd.${otherosd}.pid) | |
636 | ceph osd down osd.${otherosd} | |
637 | for o in $allosds | |
638 | do | |
639 | ceph osd out osd.$o | |
640 | done | |
11fdf7f2 TL |
641 | # Primary might change before backfill starts |
642 | sleep 2 | |
643 | primary=$(get_primary $poolname obj1) | |
b32b8144 | 644 | ceph osd unset nobackfill |
11fdf7f2 | 645 | ceph tell osd.$primary debug kick_recovery_wq 0 |
b32b8144 FG |
646 | sleep 2 |
647 | flush_pg_stats | |
648 | ||
649 | # Wait for recovery to finish | |
650 | # Can't use wait_for_clean() because state goes from active+undersized+degraded+remapped+backfilling | |
651 | # to active+undersized+remapped | |
652 | while(true) | |
653 | do | |
654 | if test "$(ceph --format json pg dump pgs | | |
11fdf7f2 | 655 | jq '.pg_stats | [.[] | .state | select(. == "incomplete")] | length')" -ne "0" |
b32b8144 FG |
656 | then |
657 | sleep 2 | |
658 | continue | |
659 | fi | |
660 | break | |
661 | done | |
662 | ceph pg dump pgs | |
663 | for i in $(seq 1 60) | |
664 | do | |
665 | if ceph pg dump pgs | grep ^$PG | grep -qv backfilling | |
666 | then | |
667 | break | |
668 | fi | |
669 | if [ $i = "60" ]; | |
670 | then | |
671 | echo "Timeout waiting for recovery to finish" | |
672 | return 1 | |
673 | fi | |
674 | sleep 1 | |
675 | done | |
676 | ||
677 | ceph pg dump pgs | |
678 | ||
b32b8144 | 679 | local misplaced=$(expr $objects \* 2) |
91327a77 | 680 | check $dir $PG $primary erasure $objects 0 $misplaced 0 || return 1 |
b32b8144 FG |
681 | |
682 | delete_pool $poolname | |
683 | kill_daemons $dir || return 1 | |
684 | } | |
685 | ||
686 | ||
687 | # [1,0,2] -> [1,3,2] | |
688 | # degraded 500 -> 0 | |
689 | # active+backfilling+degraded | |
690 | # | |
691 | # PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP | |
692 | # 1.0 500 0 500 0 0 0 100 100 active+undersized+degraded+remapped+backfilling 2017-11-06 13:57:25.412322 22'500 28:794 [1,3,2] 1 [1,NONE,2] 1 0'0 2017-11-06 13:54:58.033906 0'0 2017-11-06 13:54:58.033906 | |
693 | function TEST_backfill_ec_down_out() { | |
694 | local dir=$1 | |
695 | ||
696 | run_mon $dir a || return 1 | |
697 | run_mgr $dir x || return 1 | |
698 | run_osd $dir 0 || return 1 | |
699 | run_osd $dir 1 || return 1 | |
700 | run_osd $dir 2 || return 1 | |
701 | run_osd $dir 3 || return 1 | |
702 | run_osd $dir 4 || return 1 | |
703 | run_osd $dir 5 || return 1 | |
704 | ||
705 | ceph osd erasure-code-profile set myprofile plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd | |
706 | create_pool $poolname 1 1 erasure myprofile | |
707 | ceph osd pool set $poolname min_size 2 | |
708 | ||
709 | wait_for_clean || return 1 | |
710 | ||
711 | for i in $(seq 1 $objects) | |
712 | do | |
713 | rados -p $poolname put obj$i /dev/null | |
714 | done | |
715 | ||
716 | local PG=$(get_pg $poolname obj1) | |
717 | # Remember primary during the backfill | |
718 | local primary=$(get_primary $poolname obj1) | |
719 | local otherosd=$(get_not_primary $poolname obj1) | |
720 | ||
721 | ceph osd set nobackfill | |
722 | kill $(cat $dir/osd.${otherosd}.pid) | |
723 | ceph osd down osd.${otherosd} | |
724 | ceph osd out osd.${otherosd} | |
11fdf7f2 TL |
725 | # Primary might change before backfill starts |
726 | sleep 2 | |
727 | primary=$(get_primary $poolname obj1) | |
b32b8144 | 728 | ceph osd unset nobackfill |
11fdf7f2 | 729 | ceph tell osd.$primary debug kick_recovery_wq 0 |
b32b8144 FG |
730 | sleep 2 |
731 | ||
732 | wait_for_clean || return 1 | |
733 | ||
b32b8144 | 734 | local misplaced=$(expr $objects \* 2) |
91327a77 | 735 | check $dir $PG $primary erasure $objects 0 0 0 || return 1 |
b32b8144 FG |
736 | |
737 | delete_pool $poolname | |
738 | kill_daemons $dir || return 1 | |
739 | } | |
740 | ||
741 | ||
1adf2230 | 742 | main osd-backfill-stats "$@" |
b32b8144 FG |
743 | |
744 | # Local Variables: | |
745 | # compile-command: "make -j4 && ../qa/run-standalone.sh osd-backfill-stats.sh" | |
746 | # End: |