]>
Commit | Line | Data |
---|---|---|
1 | #! /bin/bash | |
2 | # | |
3 | # Copyright (C) 2015 Red Hat <contact@redhat.com> | |
4 | # | |
5 | # Author: David Zafman <dzafman@redhat.com> | |
6 | # | |
7 | # This program is free software; you can redistribute it and/or modify | |
8 | # it under the terms of the GNU Library Public License as published by | |
9 | # the Free Software Foundation; either version 2, or (at your option) | |
10 | # any later version. | |
11 | # | |
12 | # This program is distributed in the hope that it will be useful, | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | # GNU Library Public License for more details. | |
16 | # | |
17 | source $(dirname $0)/../detect-build-env-vars.sh | |
18 | source $CEPH_ROOT/qa/workunits/ceph-helpers.sh | |
19 | ||
20 | function run() { | |
21 | local dir=$1 | |
22 | shift | |
23 | ||
24 | export CEPH_MON="127.0.0.1:7121" # git grep '\<7121\>' : there must be only one | |
25 | export CEPH_ARGS | |
26 | CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " | |
27 | CEPH_ARGS+="--mon-host=$CEPH_MON " | |
28 | ||
29 | local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} | |
30 | for func in $funcs ; do | |
31 | $func $dir || return 1 | |
32 | done | |
33 | } | |
34 | ||
35 | function TEST_scrub_snaps() { | |
36 | local dir=$1 | |
37 | local poolname=test | |
38 | ||
39 | TESTDATA="testdata.$$" | |
40 | ||
41 | setup $dir || return 1 | |
42 | run_mon $dir a --osd_pool_default_size=1 || return 1 | |
43 | run_mgr $dir x || return 1 | |
44 | run_osd $dir 0 || return 1 | |
45 | ||
46 | wait_for_clean || return 1 | |
47 | ||
48 | # Create a pool with a single pg | |
49 | ceph osd pool create $poolname 1 1 | |
50 | poolid=$(ceph osd dump | grep "^pool.*[']test[']" | awk '{ print $2 }') | |
51 | ||
52 | dd if=/dev/urandom of=$TESTDATA bs=1032 count=1 | |
53 | for i in `seq 1 15` | |
54 | do | |
55 | rados -p $poolname put obj${i} $TESTDATA | |
56 | done | |
57 | ||
58 | SNAP=1 | |
59 | rados -p $poolname mksnap snap${SNAP} | |
60 | dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP} | |
61 | rados -p $poolname put obj1 $TESTDATA | |
62 | rados -p $poolname put obj5 $TESTDATA | |
63 | rados -p $poolname put obj3 $TESTDATA | |
64 | for i in `seq 6 14` | |
65 | do rados -p $poolname put obj${i} $TESTDATA | |
66 | done | |
67 | ||
68 | SNAP=2 | |
69 | rados -p $poolname mksnap snap${SNAP} | |
70 | dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP} | |
71 | rados -p $poolname put obj5 $TESTDATA | |
72 | ||
73 | SNAP=3 | |
74 | rados -p $poolname mksnap snap${SNAP} | |
75 | dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP} | |
76 | rados -p $poolname put obj3 $TESTDATA | |
77 | ||
78 | SNAP=4 | |
79 | rados -p $poolname mksnap snap${SNAP} | |
80 | dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP} | |
81 | rados -p $poolname put obj5 $TESTDATA | |
82 | rados -p $poolname put obj2 $TESTDATA | |
83 | ||
84 | SNAP=5 | |
85 | rados -p $poolname mksnap snap${SNAP} | |
86 | SNAP=6 | |
87 | rados -p $poolname mksnap snap${SNAP} | |
88 | dd if=/dev/urandom of=$TESTDATA bs=256 count=${SNAP} | |
89 | rados -p $poolname put obj5 $TESTDATA | |
90 | ||
91 | SNAP=7 | |
92 | rados -p $poolname mksnap snap${SNAP} | |
93 | ||
94 | rados -p $poolname rm obj4 | |
95 | rados -p $poolname rm obj2 | |
96 | ||
97 | kill_daemons $dir TERM osd || return 1 | |
98 | ||
99 | # Don't need to ceph_objectstore_tool function because osd stopped | |
100 | ||
101 | JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj1)" | |
102 | ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" --force remove | |
103 | ||
104 | JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --op list obj5 | grep \"snapid\":2)" | |
105 | ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" remove | |
106 | ||
107 | JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --op list obj5 | grep \"snapid\":1)" | |
108 | OBJ5SAVE="$JSON" | |
109 | ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" remove | |
110 | ||
111 | JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --op list obj5 | grep \"snapid\":4)" | |
112 | dd if=/dev/urandom of=$TESTDATA bs=256 count=18 | |
113 | ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" set-bytes $TESTDATA | |
114 | ||
115 | JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj3)" | |
116 | dd if=/dev/urandom of=$TESTDATA bs=256 count=15 | |
117 | ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" set-bytes $TESTDATA | |
118 | ||
119 | JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --op list obj4 | grep \"snapid\":7)" | |
120 | ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" remove | |
121 | ||
122 | JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj2)" | |
123 | ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" rm-attr snapset | |
124 | ||
125 | # Create a clone which isn't in snapset and doesn't have object info | |
126 | JSON="$(echo "$OBJ5SAVE" | sed s/snapid\":1/snapid\":7/)" | |
127 | dd if=/dev/urandom of=$TESTDATA bs=256 count=7 | |
128 | ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" set-bytes $TESTDATA | |
129 | ||
130 | rm -f $TESTDATA | |
131 | ||
132 | JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj6)" | |
133 | ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset | |
134 | JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj7)" | |
135 | ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset corrupt | |
136 | JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj8)" | |
137 | ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset seq | |
138 | JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj9)" | |
139 | ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset clone_size | |
140 | JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj10)" | |
141 | ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset clone_overlap | |
142 | JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj11)" | |
143 | ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset clones | |
144 | JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj12)" | |
145 | ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset head | |
146 | JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj13)" | |
147 | ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset snaps | |
148 | JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj14)" | |
149 | ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" clear-snapset size | |
150 | ||
151 | echo "garbage" > $dir/bad | |
152 | JSON="$(ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal --head --op list obj15)" | |
153 | ceph-objectstore-tool --data-path $dir/0 --journal-path $dir/0/journal "$JSON" set-attr snapset $dir/bad | |
154 | rm -f $dir/bad | |
155 | ||
156 | run_osd $dir 0 || return 1 | |
157 | wait_for_clean || return 1 | |
158 | ||
159 | local pgid="${poolid}.0" | |
160 | if ! pg_scrub "$pgid" ; then | |
161 | cat $dir/osd.0.log | |
162 | return 1 | |
163 | fi | |
164 | grep 'log_channel' $dir/osd.0.log | |
165 | ||
166 | rados list-inconsistent-pg $poolname > $dir/json || return 1 | |
167 | # Check pg count | |
168 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
169 | # Check pgid | |
170 | test $(jq -r '.[0]' $dir/json) = $pgid || return 1 | |
171 | ||
172 | rados list-inconsistent-snapset $pgid > $dir/json || return 1 | |
173 | test $(jq '.inconsistents | length' $dir/json) = "21" || return 1 | |
174 | ||
175 | local jqfilter='.inconsistents' | |
176 | local sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)' | |
177 | ||
178 | jq "$jqfilter" << EOF | python -c "$sortkeys" > $dir/checkcsjson | |
179 | { | |
180 | "inconsistents": [ | |
181 | { | |
182 | "errors": [ | |
183 | "headless" | |
184 | ], | |
185 | "snap": 1, | |
186 | "locator": "", | |
187 | "nspace": "", | |
188 | "name": "obj1" | |
189 | }, | |
190 | { | |
191 | "errors": [ | |
192 | "size_mismatch" | |
193 | ], | |
194 | "snap": 1, | |
195 | "locator": "", | |
196 | "nspace": "", | |
197 | "name": "obj10" | |
198 | }, | |
199 | { | |
200 | "errors": [ | |
201 | "headless" | |
202 | ], | |
203 | "snap": 1, | |
204 | "locator": "", | |
205 | "nspace": "", | |
206 | "name": "obj11" | |
207 | }, | |
208 | { | |
209 | "errors": [ | |
210 | "size_mismatch" | |
211 | ], | |
212 | "snap": 1, | |
213 | "locator": "", | |
214 | "nspace": "", | |
215 | "name": "obj14" | |
216 | }, | |
217 | { | |
218 | "errors": [ | |
219 | "headless" | |
220 | ], | |
221 | "snap": 1, | |
222 | "locator": "", | |
223 | "nspace": "", | |
224 | "name": "obj6" | |
225 | }, | |
226 | { | |
227 | "errors": [ | |
228 | "headless" | |
229 | ], | |
230 | "snap": 1, | |
231 | "locator": "", | |
232 | "nspace": "", | |
233 | "name": "obj7" | |
234 | }, | |
235 | { | |
236 | "errors": [ | |
237 | "size_mismatch" | |
238 | ], | |
239 | "snap": 1, | |
240 | "locator": "", | |
241 | "nspace": "", | |
242 | "name": "obj9" | |
243 | }, | |
244 | { | |
245 | "errors": [ | |
246 | "headless" | |
247 | ], | |
248 | "snap": 4, | |
249 | "locator": "", | |
250 | "nspace": "", | |
251 | "name": "obj2" | |
252 | }, | |
253 | { | |
254 | "errors": [ | |
255 | "size_mismatch" | |
256 | ], | |
257 | "snap": 4, | |
258 | "locator": "", | |
259 | "nspace": "", | |
260 | "name": "obj5" | |
261 | }, | |
262 | { | |
263 | "errors": [ | |
264 | "headless" | |
265 | ], | |
266 | "snap": 7, | |
267 | "locator": "", | |
268 | "nspace": "", | |
269 | "name": "obj2" | |
270 | }, | |
271 | { | |
272 | "errors": [ | |
273 | "oi_attr_missing", | |
274 | "headless" | |
275 | ], | |
276 | "snap": 7, | |
277 | "locator": "", | |
278 | "nspace": "", | |
279 | "name": "obj5" | |
280 | }, | |
281 | { | |
282 | "extra clones": [ | |
283 | 1 | |
284 | ], | |
285 | "errors": [ | |
286 | "extra_clones" | |
287 | ], | |
288 | "snap": "head", | |
289 | "locator": "", | |
290 | "nspace": "", | |
291 | "name": "obj11" | |
292 | }, | |
293 | { | |
294 | "errors": [ | |
295 | "head_mismatch" | |
296 | ], | |
297 | "snap": "head", | |
298 | "locator": "", | |
299 | "nspace": "", | |
300 | "name": "obj12" | |
301 | }, | |
302 | { | |
303 | "errors": [ | |
304 | "ss_attr_corrupted" | |
305 | ], | |
306 | "snap": "head", | |
307 | "locator": "", | |
308 | "nspace": "", | |
309 | "name": "obj15" | |
310 | }, | |
311 | { | |
312 | "extra clones": [ | |
313 | 7, | |
314 | 4 | |
315 | ], | |
316 | "errors": [ | |
317 | "ss_attr_missing", | |
318 | "extra_clones" | |
319 | ], | |
320 | "snap": "head", | |
321 | "locator": "", | |
322 | "nspace": "", | |
323 | "name": "obj2" | |
324 | }, | |
325 | { | |
326 | "errors": [ | |
327 | "size_mismatch" | |
328 | ], | |
329 | "snap": "head", | |
330 | "locator": "", | |
331 | "nspace": "", | |
332 | "name": "obj3" | |
333 | }, | |
334 | { | |
335 | "missing": [ | |
336 | 7 | |
337 | ], | |
338 | "errors": [ | |
339 | "clone_missing" | |
340 | ], | |
341 | "snap": "head", | |
342 | "locator": "", | |
343 | "nspace": "", | |
344 | "name": "obj4" | |
345 | }, | |
346 | { | |
347 | "missing": [ | |
348 | 2, | |
349 | 1 | |
350 | ], | |
351 | "extra clones": [ | |
352 | 7 | |
353 | ], | |
354 | "errors": [ | |
355 | "extra_clones", | |
356 | "clone_missing" | |
357 | ], | |
358 | "snap": "head", | |
359 | "locator": "", | |
360 | "nspace": "", | |
361 | "name": "obj5" | |
362 | }, | |
363 | { | |
364 | "extra clones": [ | |
365 | 1 | |
366 | ], | |
367 | "errors": [ | |
368 | "extra_clones" | |
369 | ], | |
370 | "snap": "head", | |
371 | "locator": "", | |
372 | "nspace": "", | |
373 | "name": "obj6" | |
374 | }, | |
375 | { | |
376 | "extra clones": [ | |
377 | 1 | |
378 | ], | |
379 | "errors": [ | |
380 | "head_mismatch", | |
381 | "extra_clones" | |
382 | ], | |
383 | "snap": "head", | |
384 | "locator": "", | |
385 | "nspace": "", | |
386 | "name": "obj7" | |
387 | }, | |
388 | { | |
389 | "errors": [ | |
390 | "snapset_mismatch" | |
391 | ], | |
392 | "snap": "head", | |
393 | "locator": "", | |
394 | "nspace": "", | |
395 | "name": "obj8" | |
396 | } | |
397 | ], | |
398 | "epoch": 20 | |
399 | } | |
400 | EOF | |
401 | ||
402 | jq "$jqfilter" $dir/json | python -c "$sortkeys" > $dir/csjson | |
403 | diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || return 1 | |
404 | ||
405 | if which jsonschema > /dev/null; | |
406 | then | |
407 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-snap.json || return 1 | |
408 | fi | |
409 | ||
410 | for i in `seq 1 7` | |
411 | do | |
412 | rados -p $poolname rmsnap snap$i | |
413 | done | |
414 | ||
415 | ERRORS=0 | |
416 | ||
417 | pidfile=$(find $dir 2>/dev/null | grep $name_prefix'[^/]*\.pid') | |
418 | pid=$(cat $pidfile) | |
419 | if ! kill -0 $pid | |
420 | then | |
421 | echo "OSD crash occurred" | |
422 | tail -100 $dir/osd.0.log | |
423 | ERRORS=$(expr $ERRORS + 1) | |
424 | fi | |
425 | ||
426 | kill_daemons $dir || return 1 | |
427 | ||
428 | declare -a err_strings | |
429 | err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj10:.* is missing in clone_overlap" | |
430 | err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj5:7 no '_' attr" | |
431 | err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj5:7 is an unexpected clone" | |
432 | err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj5:4 on disk size [(]4608[)] does not match object info size [(]512[)] adjusted for ondisk to [(]512[)]" | |
433 | err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj5:head expected clone .*:::obj5:2" | |
434 | err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj5:head expected clone .*:::obj5:1" | |
435 | err_strings[6]="log_channel[(]cluster[)] log [[]INF[]] : scrub [0-9]*[.]0 .*:::obj5:head 2 missing clone[(]s[)]" | |
436 | err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj12:head snapset.head_exists=false, but head exists" | |
437 | err_strings[8]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj8:head snaps.seq not set" | |
438 | err_strings[9]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj7:head snapset.head_exists=false, but head exists" | |
439 | err_strings[10]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj7:1 is an unexpected clone" | |
440 | err_strings[11]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj3:head on disk size [(]3840[)] does not match object info size [(]768[)] adjusted for ondisk to [(]768[)]" | |
441 | err_strings[12]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj6:1 is an unexpected clone" | |
442 | err_strings[13]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj2:head no 'snapset' attr" | |
443 | err_strings[14]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj2:7 clone ignored due to missing snapset" | |
444 | err_strings[15]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj2:4 clone ignored due to missing snapset" | |
445 | err_strings[16]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj4:head expected clone .*:::obj4:7" | |
446 | err_strings[17]="log_channel[(]cluster[)] log [[]INF[]] : scrub [0-9]*[.]0 .*:::obj4:head 1 missing clone[(]s[)]" | |
447 | err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj1:1 is an unexpected clone" | |
448 | err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj9:1 is missing in clone_size" | |
449 | err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj11:1 is an unexpected clone" | |
450 | err_strings[21]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj14:1 size 1032 != clone_size 1033" | |
451 | err_strings[22]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 23 errors" | |
452 | err_strings[23]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj15:head can't decode 'snapset' attr buffer" | |
453 | err_strings[24]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj12:1 has no oi or legacy_snaps; cannot convert 1=[[]1[]]:[[]1[]].stray_clone_snaps=[{]1=[[]1[]][}]" | |
454 | ||
455 | for i in `seq 0 ${#err_strings[@]}` | |
456 | do | |
457 | if ! grep "${err_strings[$i]}" $dir/osd.0.log > /dev/null; | |
458 | then | |
459 | echo "Missing log message '${err_strings[$i]}'" | |
460 | ERRORS=$(expr $ERRORS + 1) | |
461 | fi | |
462 | done | |
463 | ||
464 | teardown $dir || return 1 | |
465 | ||
466 | if [ $ERRORS != "0" ]; | |
467 | then | |
468 | echo "TEST FAILED WITH $ERRORS ERRORS" | |
469 | return 1 | |
470 | fi | |
471 | ||
472 | echo "TEST PASSED" | |
473 | return 0 | |
474 | } | |
475 | ||
476 | main osd-scrub-snaps "$@" | |
477 | ||
478 | # Local Variables: | |
479 | # compile-command: "cd ../.. ; make -j4 && \ | |
480 | # test/osd/osd-scrub-snaps.sh" |