]> git.proxmox.com Git - ceph.git/blob - ceph/src/test/osd/osd-scrub-repair.sh
projects / ceph.git / blob
? search:


6594e730eb07394a99af5a846438c0d339dc03f1
[ceph.git] / ceph / src / test / osd / osd-scrub-repair.sh
1 #!/bin/bash -x
2 #
3 # Copyright (C) 2014 Red Hat <contact@redhat.com>
4 #
5 # Author: Loic Dachary <loic@dachary.org>
6 #
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU Library Public License as published by
9 # the Free Software Foundation; either version 2, or (at your option)
10 # any later version.
11 #
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Library Public License for more details.
16 #
17 source $(dirname $0)/../detect-build-env-vars.sh
18 source $CEPH_ROOT/qa/workunits/ceph-helpers.sh
19
20 # Test development and debugging
21 # Set to "yes" in order to ignore diff errors and save results to update test
22 getjson="no"
23
24 # Ignore the epoch and filter out the attr '_' value because it has date information and won't match
25 jqfilter='.inconsistents | (.[].shards[].attrs[] | select(.name == "_") | .value) |= "----Stripped-by-test----"'
26 sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)'
27
28 # Remove items are not consistent across runs, the pg interval and client
29 sedfilter='s/\([ ]*\"\(selected_\)*object_info\":.*head[(]\)[^[:space:]]* [^[:space:]]* \(.*\)/\1\3/'
30
31 function run() {
32 local dir=$1
33 shift
34
35 export CEPH_MON="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one
36 export CEPH_ARGS
37 CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
38 CEPH_ARGS+="--mon-host=$CEPH_MON "
39
40 local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
41 for func in $funcs ; do
42 $func $dir || return 1
43 done
44 }
45
46 function add_something() {
47 local dir=$1
48 local poolname=$2
49 local obj=${3:-SOMETHING}
50 local scrub=${4:-noscrub}
51
52 if [ "$scrub" = "noscrub" ];
53 then
54 ceph osd set noscrub || return 1
55 ceph osd set nodeep-scrub || return 1
56 else
57 ceph osd unset noscrub || return 1
58 ceph osd unset nodeep-scrub || return 1
59 fi
60
61 local payload=ABCDEF
62 echo $payload > $dir/ORIGINAL
63 rados --pool $poolname put $obj $dir/ORIGINAL || return 1
64 }
65
66 #
67 # Corrupt one copy of a replicated pool
68 #
69 function TEST_corrupt_and_repair_replicated() {
70 local dir=$1
71 local poolname=rbd
72
73 setup $dir || return 1
74 run_mon $dir a --osd_pool_default_size=2 || return 1
75 run_mgr $dir x || return 1
76 run_osd $dir 0 || return 1
77 run_osd $dir 1 || return 1
78 wait_for_clean || return 1
79
80 add_something $dir $poolname || return 1
81 corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1
82 # Reproduces http://tracker.ceph.com/issues/8914
83 corrupt_and_repair_one $dir $poolname $(get_primary $poolname SOMETHING) || return 1
84
85 teardown $dir || return 1
86 }
87
88 function corrupt_and_repair_two() {
89 local dir=$1
90 local poolname=$2
91 local first=$3
92 local second=$4
93
94 #
95 # 1) remove the corresponding file from the OSDs
96 #
97 pids=""
98 run_in_background pids objectstore_tool $dir $first SOMETHING remove
99 run_in_background pids objectstore_tool $dir $second SOMETHING remove
100 wait_background pids
101 return_code=$?
102 if [ $return_code -ne 0 ]; then return $return_code; fi
103
104 #
105 # 2) repair the PG
106 #
107 local pg=$(get_pg $poolname SOMETHING)
108 repair $pg
109 #
110 # 3) The files must be back
111 #
112 pids=""
113 run_in_background pids objectstore_tool $dir $first SOMETHING list-attrs
114 run_in_background pids objectstore_tool $dir $second SOMETHING list-attrs
115 wait_background pids
116 return_code=$?
117 if [ $return_code -ne 0 ]; then return $return_code; fi
118
119 rados --pool $poolname get SOMETHING $dir/COPY || return 1
120 diff $dir/ORIGINAL $dir/COPY || return 1
121 }
122
123 #
124 # 1) add an object
125 # 2) remove the corresponding file from a designated OSD
126 # 3) repair the PG
127 # 4) check that the file has been restored in the designated OSD
128 #
129 function corrupt_and_repair_one() {
130 local dir=$1
131 local poolname=$2
132 local osd=$3
133
134 #
135 # 1) remove the corresponding file from the OSD
136 #
137 objectstore_tool $dir $osd SOMETHING remove || return 1
138 #
139 # 2) repair the PG
140 #
141 local pg=$(get_pg $poolname SOMETHING)
142 repair $pg
143 #
144 # 3) The file must be back
145 #
146 objectstore_tool $dir $osd SOMETHING list-attrs || return 1
147 rados --pool $poolname get SOMETHING $dir/COPY || return 1
148 diff $dir/ORIGINAL $dir/COPY || return 1
149 }
150
151 function corrupt_and_repair_erasure_coded() {
152 local dir=$1
153 local poolname=$2
154
155 add_something $dir $poolname || return 1
156
157 local primary=$(get_primary $poolname SOMETHING)
158 local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
159 local not_primary_first=${osds[0]}
160 local not_primary_second=${osds[1]}
161
162 # Reproduces http://tracker.ceph.com/issues/10017
163 corrupt_and_repair_one $dir $poolname $primary || return 1
164 # Reproduces http://tracker.ceph.com/issues/10409
165 corrupt_and_repair_one $dir $poolname $not_primary_first || return 1
166 corrupt_and_repair_two $dir $poolname $not_primary_first $not_primary_second || return 1
167 corrupt_and_repair_two $dir $poolname $primary $not_primary_first || return 1
168
169 }
170
171 function create_ec_pool() {
172 local pool_name=$1
173 local allow_overwrites=$2
174
175 ceph osd erasure-code-profile set myprofile ruleset-failure-domain=osd $3 $4 $5 $6 $7 || return 1
176
177 ceph osd pool create "$poolname" 1 1 erasure myprofile || return 1
178
179 if [ "$allow_overwrites" = "true" ]; then
180 ceph osd pool set "$poolname" allow_ec_overwrites true || return 1
181 fi
182
183 wait_for_clean || return 1
184 return 0
185 }
186
187 function auto_repair_erasure_coded() {
188 local dir=$1
189 local allow_overwrites=$2
190 local poolname=ecpool
191
192 # Launch a cluster with 5 seconds scrub interval
193 setup $dir || return 1
194 run_mon $dir a || return 1
195 run_mgr $dir x || return 1
196 local ceph_osd_args="--osd-scrub-auto-repair=true \
197 --osd-deep-scrub-interval=5 \
198 --osd-scrub-max-interval=5 \
199 --osd-scrub-min-interval=5 \
200 --osd-scrub-interval-randomize-ratio=0"
201 for id in $(seq 0 2) ; do
202 if [ "$allow_overwrites" = "true" ]; then
203 run_osd_bluestore $dir $id $ceph_osd_args || return 1
204 else
205 run_osd $dir $id $ceph_osd_args || return 1
206 fi
207 done
208 wait_for_clean || return 1
209
210 # Create an EC pool
211 create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
212
213 # Put an object
214 local payload=ABCDEF
215 echo $payload > $dir/ORIGINAL
216 rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
217
218 # Remove the object from one shard physically
219 # Restarted osd get $ceph_osd_args passed
220 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
221 # Wait for auto repair
222 local pgid=$(get_pg $poolname SOMETHING)
223 wait_for_scrub $pgid "$(get_last_scrub_stamp $pgid)"
224 wait_for_clean || return 1
225 # Verify - the file should be back
226 # Restarted osd get $ceph_osd_args passed
227 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
228 rados --pool $poolname get SOMETHING $dir/COPY || return 1
229 diff $dir/ORIGINAL $dir/COPY || return 1
230
231 # Tear down
232 teardown $dir || return 1
233 }
234
235 function TEST_auto_repair_erasure_coded_appends() {
236 auto_repair_erasure_coded $1 false
237 }
238
239 function TEST_auto_repair_erasure_coded_overwrites() {
240 auto_repair_erasure_coded $1 true
241 }
242
243 function corrupt_and_repair_jerasure() {
244 local dir=$1
245 local allow_overwrites=$2
246 local poolname=ecpool
247
248 setup $dir || return 1
249 run_mon $dir a || return 1
250 run_mgr $dir x || return 1
251 for id in $(seq 0 3) ; do
252 if [ "$allow_overwrites" = "true" ]; then
253 run_osd_bluestore $dir $id || return 1
254 else
255 run_osd $dir $id || return 1
256 fi
257 done
258 wait_for_clean || return 1
259
260 create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
261 corrupt_and_repair_erasure_coded $dir $poolname || return 1
262
263 teardown $dir || return 1
264 }
265
266 function TEST_corrupt_and_repair_jerasure_appends() {
267 corrupt_and_repair_jerasure $1
268 }
269
270 function TEST_corrupt_and_repair_jerasure_overwrites() {
271 corrupt_and_repair_jerasure $1 true
272 }
273
274 function corrupt_and_repair_lrc() {
275 local dir=$1
276 local allow_overwrites=$2
277 local poolname=ecpool
278
279 setup $dir || return 1
280 run_mon $dir a || return 1
281 run_mgr $dir x || return 1
282 for id in $(seq 0 9) ; do
283 if [ "$allow_overwrites" = "true" ]; then
284 run_osd_bluestore $dir $id || return 1
285 else
286 run_osd $dir $id || return 1
287 fi
288 done
289 wait_for_clean || return 1
290
291 create_ec_pool $poolname $allow_overwrites k=4 m=2 l=3 plugin=lrc || return 1
292 corrupt_and_repair_erasure_coded $dir $poolname || return 1
293
294 teardown $dir || return 1
295 }
296
297 function TEST_corrupt_and_repair_lrc_appends() {
298 corrupt_and_repair_jerasure $1
299 }
300
301 function TEST_corrupt_and_repair_lrc_overwrites() {
302 corrupt_and_repair_jerasure $1 true
303 }
304
305 function unfound_erasure_coded() {
306 local dir=$1
307 local allow_overwrites=$2
308 local poolname=ecpool
309 local payload=ABCDEF
310
311 setup $dir || return 1
312 run_mon $dir a || return 1
313 run_mgr $dir x || return 1
314 for id in $(seq 0 3) ; do
315 if [ "$allow_overwrites" = "true" ]; then
316 run_osd_bluestore $dir $id || return 1
317 else
318 run_osd $dir $id || return 1
319 fi
320 done
321 wait_for_clean || return 1
322
323 create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
324
325 add_something $dir $poolname || return 1
326
327 local primary=$(get_primary $poolname SOMETHING)
328 local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
329 local not_primary_first=${osds[0]}
330 local not_primary_second=${osds[1]}
331 local not_primary_third=${osds[2]}
332
333 #
334 # 1) remove the corresponding file from the OSDs
335 #
336 pids=""
337 run_in_background pids objectstore_tool $dir $not_primary_first SOMETHING remove
338 run_in_background pids objectstore_tool $dir $not_primary_second SOMETHING remove
339 run_in_background pids objectstore_tool $dir $not_primary_third SOMETHING remove
340 wait_background pids
341 return_code=$?
342 if [ $return_code -ne 0 ]; then return $return_code; fi
343
344 #
345 # 2) repair the PG
346 #
347 local pg=$(get_pg $poolname SOMETHING)
348 repair $pg
349 #
350 # 3) check pg state
351 #
352 # it may take a bit to appear due to mon/mgr asynchrony
353 for f in `seq 1 60`; do
354 ceph -s | grep "1/1 unfound" && break
355 sleep 1
356 done
357 ceph -s|grep "4 osds: 4 up, 4 in" || return 1
358 ceph -s|grep "1/1 unfound" || return 1
359
360 teardown $dir || return 1
361 }
362
363 function TEST_unfound_erasure_coded_appends() {
364 unfound_erasure_coded $1
365 }
366
367 function TEST_unfound_erasure_coded_overwrites() {
368 unfound_erasure_coded $1 true
369 }
370
371 #
372 # list_missing for EC pool
373 #
374 function list_missing_erasure_coded() {
375 local dir=$1
376 local allow_overwrites=$2
377 local poolname=ecpool
378
379 setup $dir || return 1
380 run_mon $dir a || return 1
381 run_mgr $dir x || return 1
382 for id in $(seq 0 2) ; do
383 if [ "$allow_overwrites" = "true" ]; then
384 run_osd_bluestore $dir $id || return 1
385 else
386 run_osd $dir $id || return 1
387 fi
388 done
389 wait_for_clean || return 1
390
391 create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
392
393 # Put an object and remove the two shards (including primary)
394 add_something $dir $poolname MOBJ0 || return 1
395 local -a osds0=($(get_osds $poolname MOBJ0))
396
397 # Put another object and remove two shards (excluding primary)
398 add_something $dir $poolname MOBJ1 || return 1
399 local -a osds1=($(get_osds $poolname MOBJ1))
400
401 # Stop all osd daemons
402 for id in $(seq 0 2) ; do
403 kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1
404 done
405
406 id=${osds0[0]}
407 ceph-objectstore-tool --data-path $dir/$id \
408 MOBJ0 remove || return 1
409 id=${osds0[1]}
410 ceph-objectstore-tool --data-path $dir/$id \
411 MOBJ0 remove || return 1
412
413 id=${osds1[1]}
414 ceph-objectstore-tool --data-path $dir/$id \
415 MOBJ1 remove || return 1
416 id=${osds1[2]}
417 ceph-objectstore-tool --data-path $dir/$id \
418 MOBJ1 remove || return 1
419
420 for id in $(seq 0 2) ; do
421 activate_osd $dir $id >&2 || return 1
422 done
423 wait_for_clean || return 1
424
425 # Get get - both objects should in the same PG
426 local pg=$(get_pg $poolname MOBJ0)
427
428 # Repair the PG, which triggers the recovering,
429 # and should mark the object as unfound
430 repair $pg
431
432 for i in $(seq 0 120) ; do
433 [ $i -lt 60 ] || return 1
434 matches=$(ceph pg $pg list_missing | egrep "MOBJ0|MOBJ1" | wc -l)
435 [ $matches -eq 2 ] && break
436 done
437
438 teardown $dir || return 1
439 }
440
441 function TEST_list_missing_erasure_coded_appends() {
442 list_missing_erasure_coded $1 false
443 }
444
445 function TEST_list_missing_erasure_coded_overwrites() {
446 list_missing_erasure_coded $1 true
447 }
448
449 #
450 # Corrupt one copy of a replicated pool
451 #
452 function TEST_corrupt_scrub_replicated() {
453 local dir=$1
454 local poolname=csr_pool
455 local total_objs=15
456
457 setup $dir || return 1
458 run_mon $dir a --osd_pool_default_size=2 || return 1
459 run_mgr $dir x || return 1
460 run_osd $dir 0 || return 1
461 run_osd $dir 1 || return 1
462 wait_for_clean || return 1
463
464 ceph osd pool create $poolname 1 1 || return 1
465 wait_for_clean || return 1
466
467 for i in $(seq 1 $total_objs) ; do
468 objname=ROBJ${i}
469 add_something $dir $poolname $objname || return 1
470
471 rados --pool $poolname setomapheader $objname hdr-$objname || return 1
472 rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1
473 done
474
475 local pg=$(get_pg $poolname ROBJ0)
476
477 # Compute an old omap digest and save oi
478 CEPH_ARGS='' ceph daemon $dir//ceph-osd.0.asok \
479 config set osd_deep_scrub_update_digest_min_age 0
480 CEPH_ARGS='' ceph daemon $dir//ceph-osd.1.asok \
481 config set osd_deep_scrub_update_digest_min_age 0
482 pg_deep_scrub $pg
483
484 for i in $(seq 1 $total_objs) ; do
485 objname=ROBJ${i}
486
487 # Alternate corruption between osd.0 and osd.1
488 local osd=$(expr $i % 2)
489
490 case $i in
491 1)
492 # Size (deep scrub data_digest too)
493 local payload=UVWXYZZZ
494 echo $payload > $dir/CORRUPT
495 objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
496 ;;
497
498 2)
499 # digest (deep scrub only)
500 local payload=UVWXYZ
501 echo $payload > $dir/CORRUPT
502 objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
503 ;;
504
505 3)
506 # missing
507 objectstore_tool $dir $osd $objname remove || return 1
508 ;;
509
510 4)
511 # Modify omap value (deep scrub only)
512 objectstore_tool $dir $osd $objname set-omap key-$objname $dir/CORRUPT || return 1
513 ;;
514
515 5)
516 # Delete omap key (deep scrub only)
517 objectstore_tool $dir $osd $objname rm-omap key-$objname || return 1
518 ;;
519
520 6)
521 # Add extra omap key (deep scrub only)
522 echo extra > $dir/extra-val
523 objectstore_tool $dir $osd $objname set-omap key2-$objname $dir/extra-val || return 1
524 rm $dir/extra-val
525 ;;
526
527 7)
528 # Modify omap header (deep scrub only)
529 echo -n newheader > $dir/hdr
530 objectstore_tool $dir $osd $objname set-omaphdr $dir/hdr || return 1
531 rm $dir/hdr
532 ;;
533
534 8)
535 rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
536 rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
537
538 # Break xattrs
539 echo -n bad-val > $dir/bad-val
540 objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
541 objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
542 echo -n val3-$objname > $dir/newval
543 objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
544 rm $dir/bad-val $dir/newval
545 ;;
546
547 9)
548 objectstore_tool $dir $osd $objname get-attr _ > $dir/robj9-oi
549 echo -n D > $dir/change
550 rados --pool $poolname put $objname $dir/change
551 objectstore_tool $dir $osd $objname set-attr _ $dir/robj9-oi
552 rm $dir/oi $dir/change
553 ;;
554
555 # ROBJ10 must be handled after digests are re-computed by a deep scrub below
556 # ROBJ11 must be handled with config change before deep scrub
557 # ROBJ12 must be handled with config change before scrubs
558 # ROBJ13 must be handled before scrubs
559
560 14)
561 echo -n bad-val > $dir/bad-val
562 objectstore_tool $dir 0 $objname set-attr _ $dir/bad-val || return 1
563 objectstore_tool $dir 1 $objname rm-attr _ || return 1
564 rm $dir/bad-val
565 ;;
566
567 15)
568 objectstore_tool $dir $osd $objname rm-attr _ || return 1
569
570 esac
571 done
572
573 local pg=$(get_pg $poolname ROBJ0)
574
575 set_config osd 0 filestore_debug_inject_read_err true || return 1
576 set_config osd 1 filestore_debug_inject_read_err true || return 1
577 CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.1.asok \
578 injectdataerr $poolname ROBJ11 || return 1
579 CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok \
580 injectmdataerr $poolname ROBJ12 || return 1
581 CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok \
582 injectmdataerr $poolname ROBJ13 || return 1
583 CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.1.asok \
584 injectdataerr $poolname ROBJ13 || return 1
585
586 pg_scrub $pg
587
588 rados list-inconsistent-pg $poolname > $dir/json || return 1
589 # Check pg count
590 test $(jq '. | length' $dir/json) = "1" || return 1
591 # Check pgid
592 test $(jq -r '.[0]' $dir/json) = $pg || return 1
593
594 rados list-inconsistent-obj $pg > $dir/json || return 1
595 # Get epoch for repair-get requests
596 epoch=$(jq .epoch $dir/json)
597
598 jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
599 {
600 "inconsistents": [
601 {
602 "shards": [
603 {
604 "size": 7,
605 "errors": [],
606 "osd": 0
607 },
608 {
609 "size": 9,
610 "errors": [
611 "size_mismatch_oi"
612 ],
613 "osd": 1
614 }
615 ],
616 "selected_object_info": "2:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])",
617 "union_shard_errors": [
618 "size_mismatch_oi"
619 ],
620 "errors": [
621 "size_mismatch"
622 ],
623 "object": {
624 "version": 3,
625 "snap": "head",
626 "locator": "",
627 "nspace": "",
628 "name": "ROBJ1"
629 }
630 },
631 {
632 "shards": [
633 {
634 "errors": [
635 "stat_error"
636 ],
637 "osd": 0
638 },
639 {
640 "size": 7,
641 "errors": [],
642 "osd": 1
643 }
644 ],
645 "selected_object_info": "2:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])",
646 "union_shard_errors": [
647 "stat_error"
648 ],
649 "errors": [],
650 "object": {
651 "version": 36,
652 "snap": "head",
653 "locator": "",
654 "nspace": "",
655 "name": "ROBJ12"
656 }
657 },
658 {
659 "shards": [
660 {
661 "errors": [
662 "stat_error"
663 ],
664 "osd": 0
665 },
666 {
667 "size": 7,
668 "errors": [],
669 "osd": 1
670 }
671 ],
672 "selected_object_info": "2:d60617f9:::ROBJ13:head(47'55 osd.0.0:54 dirty|omap|data_digest|omap_digest s 7 uv 39 dd 2ddbf8f5 od 6441854d alloc_hint [0 0 0])",
673 "union_shard_errors": [
674 "stat_error"
675 ],
676 "errors": [],
677 "object": {
678 "version": 39,
679 "snap": "head",
680 "locator": "",
681 "nspace": "",
682 "name": "ROBJ13"
683 }
684 },
685 {
686 "shards": [
687 {
688 "size": 7,
689 "errors": [
690 "oi_attr_corrupted"
691 ],
692 "osd": 0
693 },
694 {
695 "size": 7,
696 "errors": [
697 "oi_attr_missing"
698 ],
699 "osd": 1
700 }
701 ],
702 "union_shard_errors": [
703 "oi_attr_missing",
704 "oi_attr_corrupted"
705 ],
706 "errors": [],
707 "object": {
708 "version": 0,
709 "snap": "head",
710 "locator": "",
711 "nspace": "",
712 "name": "ROBJ14"
713 }
714 },
715 {
716 "shards": [
717 {
718 "attrs": [
719 {
720 "Base64": true,
721 "value": "",
722 "name": "_"
723 },
724 {
725 "Base64": true,
726 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
727 "name": "snapset"
728 }
729 ],
730 "size": 7,
731 "errors": [],
732 "osd": 0
733 },
734 {
735 "attrs": [
736 {
737 "Base64": true,
738 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
739 "name": "snapset"
740 }
741 ],
742 "size": 7,
743 "errors": [
744 "oi_attr_missing"
745 ],
746 "osd": 1
747 }
748 ],
749 "selected_object_info": "2:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])",
750 "union_shard_errors": [
751 "oi_attr_missing"
752 ],
753 "errors": [
754 "attr_name_mismatch"
755 ],
756 "object": {
757 "version": 45,
758 "snap": "head",
759 "locator": "",
760 "nspace": "",
761 "name": "ROBJ15"
762 }
763 },
764 {
765 "shards": [
766 {
767 "size": 7,
768 "errors": [],
769 "osd": 0
770 },
771 {
772 "errors": [
773 "missing"
774 ],
775 "osd": 1
776 }
777 ],
778 "selected_object_info": "2:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])",
779 "union_shard_errors": [
780 "missing"
781 ],
782 "errors": [],
783 "object": {
784 "version": 9,
785 "snap": "head",
786 "locator": "",
787 "nspace": "",
788 "name": "ROBJ3"
789 }
790 },
791 {
792 "shards": [
793 {
794 "attrs": [
795 {
796 "Base64": true,
797 "value": "",
798 "name": "_"
799 },
800 {
801 "Base64": false,
802 "value": "bad-val",
803 "name": "_key1-ROBJ8"
804 },
805 {
806 "Base64": false,
807 "value": "val3-ROBJ8",
808 "name": "_key3-ROBJ8"
809 },
810 {
811 "Base64": true,
812 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
813 "name": "snapset"
814 }
815 ],
816 "size": 7,
817 "errors": [],
818 "osd": 0
819 },
820 {
821 "attrs": [
822 {
823 "Base64": true,
824 "value": "",
825 "name": "_"
826 },
827 {
828 "Base64": false,
829 "value": "val1-ROBJ8",
830 "name": "_key1-ROBJ8"
831 },
832 {
833 "Base64": false,
834 "value": "val2-ROBJ8",
835 "name": "_key2-ROBJ8"
836 },
837 {
838 "Base64": true,
839 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
840 "name": "snapset"
841 }
842 ],
843 "size": 7,
844 "errors": [],
845 "osd": 1
846 }
847 ],
848 "selected_object_info": "2:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 62 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])",
849 "union_shard_errors": [],
850 "errors": [
851 "attr_value_mismatch",
852 "attr_name_mismatch"
853 ],
854 "object": {
855 "version": 62,
856 "snap": "head",
857 "locator": "",
858 "nspace": "",
859 "name": "ROBJ8"
860 }
861 },
862 {
863 "shards": [
864 {
865 "attrs": [
866 {
867 "Base64": true,
868 "value": "",
869 "name": "_"
870 },
871 {
872 "Base64": true,
873 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
874 "name": "snapset"
875 }
876 ],
877 "object_info": "2:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 63 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])",
878 "size": 1,
879 "errors": [],
880 "osd": 0
881 },
882 {
883 "attrs": [
884 {
885 "Base64": true,
886 "value": "",
887 "name": "_"
888 },
889 {
890 "Base64": true,
891 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
892 "name": "snapset"
893 }
894 ],
895 "object_info": "2:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])",
896 "size": 1,
897 "errors": [],
898 "osd": 1
899 }
900 ],
901 "selected_object_info": "2:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 63 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])",
902 "union_shard_errors": [],
903 "errors": [
904 "object_info_inconsistency",
905 "attr_value_mismatch"
906 ],
907 "object": {
908 "version": 63,
909 "snap": "head",
910 "locator": "",
911 "nspace": "",
912 "name": "ROBJ9"
913 }
914 }
915 ],
916 "epoch": 0
917 }
918 EOF
919
920 jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson
921 diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
922 if test $getjson = "yes"
923 then
924 jq '.' $dir/json > save1.json
925 fi
926
927 if which jsonschema > /dev/null;
928 then
929 jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
930 fi
931
932 objname=ROBJ9
933 # Change data and size again because digest was recomputed
934 echo -n ZZZ > $dir/change
935 rados --pool $poolname put $objname $dir/change
936 # Set one to an even older value
937 objectstore_tool $dir 0 $objname set-attr _ $dir/robj9-oi
938 rm $dir/oi $dir/change
939
940 objname=ROBJ10
941 objectstore_tool $dir 1 $objname get-attr _ > $dir/oi
942 rados --pool $poolname setomapval $objname key2-$objname val2-$objname
943 objectstore_tool $dir 0 $objname set-attr _ $dir/oi
944 objectstore_tool $dir 1 $objname set-attr _ $dir/oi
945 rm $dir/oi
946
947 set_config osd 0 filestore_debug_inject_read_err true || return 1
948 set_config osd 1 filestore_debug_inject_read_err true || return 1
949 CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.1.asok \
950 injectdataerr $poolname ROBJ11 || return 1
951 CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok \
952 injectmdataerr $poolname ROBJ12 || return 1
953 CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok \
954 injectmdataerr $poolname ROBJ13 || return 1
955 CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.1.asok \
956 injectdataerr $poolname ROBJ13 || return 1
957 pg_deep_scrub $pg
958
959 rados list-inconsistent-pg $poolname > $dir/json || return 1
960 # Check pg count
961 test $(jq '. | length' $dir/json) = "1" || return 1
962 # Check pgid
963 test $(jq -r '.[0]' $dir/json) = $pg || return 1
964
965 rados list-inconsistent-obj $pg > $dir/json || return 1
966 # Get epoch for repair-get requests
967 epoch=$(jq .epoch $dir/json)
968
969 jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
970 {
971 "inconsistents": [
972 {
973 "shards": [
974 {
975 "data_digest": "0x2ddbf8f5",
976 "omap_digest": "0xf5fba2c6",
977 "size": 7,
978 "errors": [],
979 "osd": 0
980 },
981 {
982 "data_digest": "0x2d4a11c2",
983 "omap_digest": "0xf5fba2c6",
984 "size": 9,
985 "errors": [
986 "data_digest_mismatch_oi",
987 "size_mismatch_oi"
988 ],
989 "osd": 1
990 }
991 ],
992 "selected_object_info": "2:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])",
993 "union_shard_errors": [
994 "data_digest_mismatch_oi",
995 "size_mismatch_oi"
996 ],
997 "errors": [
998 "data_digest_mismatch",
999 "size_mismatch"
1000 ],
1001 "object": {
1002 "version": 3,
1003 "snap": "head",
1004 "locator": "",
1005 "nspace": "",
1006 "name": "ROBJ1"
1007 }
1008 },
1009 {
1010 "shards": [
1011 {
1012 "data_digest": "0x2ddbf8f5",
1013 "omap_digest": "0xa8dd5adc",
1014 "size": 7,
1015 "errors": [
1016 "omap_digest_mismatch_oi"
1017 ],
1018 "osd": 0
1019 },
1020 {
1021 "data_digest": "0x2ddbf8f5",
1022 "omap_digest": "0xa8dd5adc",
1023 "size": 7,
1024 "errors": [
1025 "omap_digest_mismatch_oi"
1026 ],
1027 "osd": 1
1028 }
1029 ],
1030 "selected_object_info": "2:b1f19cbd:::ROBJ10:head(47'51 osd.0.0:50 dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [0 0 0])",
1031 "union_shard_errors": [
1032 "omap_digest_mismatch_oi"
1033 ],
1034 "errors": [],
1035 "object": {
1036 "version": 30,
1037 "snap": "head",
1038 "locator": "",
1039 "nspace": "",
1040 "name": "ROBJ10"
1041 }
1042 },
1043 {
1044 "shards": [
1045 {
1046 "data_digest": "0x2ddbf8f5",
1047 "omap_digest": "0xa03cef03",
1048 "size": 7,
1049 "errors": [],
1050 "osd": 0
1051 },
1052 {
1053 "size": 7,
1054 "errors": [
1055 "read_error"
1056 ],
1057 "osd": 1
1058 }
1059 ],
1060 "selected_object_info": "2:87abbf36:::ROBJ11:head(47'48 osd.0.0:47 dirty|omap|data_digest|omap_digest s 7 uv 33 dd 2ddbf8f5 od a03cef03 alloc_hint [0 0 0])",
1061 "union_shard_errors": [
1062 "read_error"
1063 ],
1064 "errors": [],
1065 "object": {
1066 "version": 33,
1067 "snap": "head",
1068 "locator": "",
1069 "nspace": "",
1070 "name": "ROBJ11"
1071 }
1072 },
1073 {
1074 "shards": [
1075 {
1076 "errors": [
1077 "stat_error"
1078 ],
1079 "osd": 0
1080 },
1081 {
1082 "data_digest": "0x2ddbf8f5",
1083 "omap_digest": "0x067f306a",
1084 "size": 7,
1085 "errors": [],
1086 "osd": 1
1087 }
1088 ],
1089 "selected_object_info": "2:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])",
1090 "union_shard_errors": [
1091 "stat_error"
1092 ],
1093 "errors": [],
1094 "object": {
1095 "version": 36,
1096 "snap": "head",
1097 "locator": "",
1098 "nspace": "",
1099 "name": "ROBJ12"
1100 }
1101 },
1102 {
1103 "shards": [
1104 {
1105 "errors": [
1106 "stat_error"
1107 ],
1108 "osd": 0
1109 },
1110 {
1111 "size": 7,
1112 "errors": [
1113 "read_error"
1114 ],
1115 "osd": 1
1116 }
1117 ],
1118 "union_shard_errors": [
1119 "stat_error",
1120 "read_error"
1121 ],
1122 "errors": [],
1123 "object": {
1124 "version": 0,
1125 "snap": "head",
1126 "locator": "",
1127 "nspace": "",
1128 "name": "ROBJ13"
1129 }
1130 },
1131 {
1132 "shards": [
1133 {
1134 "data_digest": "0x2ddbf8f5",
1135 "omap_digest": "0x4f14f849",
1136 "size": 7,
1137 "errors": [
1138 "oi_attr_corrupted"
1139 ],
1140 "osd": 0
1141 },
1142 {
1143 "data_digest": "0x2ddbf8f5",
1144 "omap_digest": "0x4f14f849",
1145 "size": 7,
1146 "errors": [
1147 "oi_attr_missing"
1148 ],
1149 "osd": 1
1150 }
1151 ],
1152 "union_shard_errors": [
1153 "oi_attr_missing",
1154 "oi_attr_corrupted"
1155 ],
1156 "errors": [],
1157 "object": {
1158 "version": 0,
1159 "snap": "head",
1160 "locator": "",
1161 "nspace": "",
1162 "name": "ROBJ14"
1163 }
1164 },
1165 {
1166 "shards": [
1167 {
1168 "attrs": [
1169 {
1170 "Base64": true,
1171 "value": "",
1172 "name": "_"
1173 },
1174 {
1175 "Base64": true,
1176 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1177 "name": "snapset"
1178 }
1179 ],
1180 "data_digest": "0x2ddbf8f5",
1181 "omap_digest": "0x2d2a4d6e",
1182 "size": 7,
1183 "errors": [],
1184 "osd": 0
1185 },
1186 {
1187 "attrs": [
1188 {
1189 "Base64": true,
1190 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1191 "name": "snapset"
1192 }
1193 ],
1194 "data_digest": "0x2ddbf8f5",
1195 "omap_digest": "0x2d2a4d6e",
1196 "size": 7,
1197 "errors": [
1198 "oi_attr_missing"
1199 ],
1200 "osd": 1
1201 }
1202 ],
1203 "selected_object_info": "2:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])",
1204 "union_shard_errors": [
1205 "oi_attr_missing"
1206 ],
1207 "errors": [
1208 "attr_name_mismatch"
1209 ],
1210 "object": {
1211 "version": 45,
1212 "snap": "head",
1213 "locator": "",
1214 "nspace": "",
1215 "name": "ROBJ15"
1216 }
1217 },
1218 {
1219 "shards": [
1220 {
1221 "data_digest": "0x578a4830",
1222 "omap_digest": "0xf8e11918",
1223 "size": 7,
1224 "errors": [
1225 "data_digest_mismatch_oi"
1226 ],
1227 "osd": 0
1228 },
1229 {
1230 "data_digest": "0x2ddbf8f5",
1231 "omap_digest": "0xf8e11918",
1232 "size": 7,
1233 "errors": [],
1234 "osd": 1
1235 }
1236 ],
1237 "selected_object_info": "2:e97ce31e:::ROBJ2:head(47'56 osd.0.0:55 dirty|omap|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od f8e11918 alloc_hint [0 0 0])",
1238 "union_shard_errors": [
1239 "data_digest_mismatch_oi"
1240 ],
1241 "errors": [
1242 "data_digest_mismatch"
1243 ],
1244 "object": {
1245 "version": 6,
1246 "snap": "head",
1247 "locator": "",
1248 "nspace": "",
1249 "name": "ROBJ2"
1250 }
1251 },
1252 {
1253 "shards": [
1254 {
1255 "data_digest": "0x2ddbf8f5",
1256 "omap_digest": "0x00b35dfd",
1257 "size": 7,
1258 "errors": [],
1259 "osd": 0
1260 },
1261 {
1262 "errors": [
1263 "missing"
1264 ],
1265 "osd": 1
1266 }
1267 ],
1268 "selected_object_info": "2:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])",
1269 "union_shard_errors": [
1270 "missing"
1271 ],
1272 "errors": [],
1273 "object": {
1274 "version": 9,
1275 "snap": "head",
1276 "locator": "",
1277 "nspace": "",
1278 "name": "ROBJ3"
1279 }
1280 },
1281 {
1282 "shards": [
1283 {
1284 "data_digest": "0x2ddbf8f5",
1285 "omap_digest": "0xd7178dfe",
1286 "size": 7,
1287 "errors": [
1288 "omap_digest_mismatch_oi"
1289 ],
1290 "osd": 0
1291 },
1292 {
1293 "data_digest": "0x2ddbf8f5",
1294 "omap_digest": "0xe2d46ea4",
1295 "size": 7,
1296 "errors": [],
1297 "osd": 1
1298 }
1299 ],
1300 "selected_object_info": "2:f4981d31:::ROBJ4:head(47'58 osd.0.0:57 dirty|omap|data_digest|omap_digest s 7 uv 12 dd 2ddbf8f5 od e2d46ea4 alloc_hint [0 0 0])",
1301 "union_shard_errors": [
1302 "omap_digest_mismatch_oi"
1303 ],
1304 "errors": [
1305 "omap_digest_mismatch"
1306 ],
1307 "object": {
1308 "version": 12,
1309 "snap": "head",
1310 "locator": "",
1311 "nspace": "",
1312 "name": "ROBJ4"
1313 }
1314 },
1315 {
1316 "shards": [
1317 {
1318 "data_digest": "0x2ddbf8f5",
1319 "omap_digest": "0x1a862a41",
1320 "size": 7,
1321 "errors": [],
1322 "osd": 0
1323 },
1324 {
1325 "data_digest": "0x2ddbf8f5",
1326 "omap_digest": "0x06cac8f6",
1327 "size": 7,
1328 "errors": [
1329 "omap_digest_mismatch_oi"
1330 ],
1331 "osd": 1
1332 }
1333 ],
1334 "selected_object_info": "2:f4bfd4d1:::ROBJ5:head(47'59 osd.0.0:58 dirty|omap|data_digest|omap_digest s 7 uv 15 dd 2ddbf8f5 od 1a862a41 alloc_hint [0 0 0])",
1335 "union_shard_errors": [
1336 "omap_digest_mismatch_oi"
1337 ],
1338 "errors": [
1339 "omap_digest_mismatch"
1340 ],
1341 "object": {
1342 "version": 15,
1343 "snap": "head",
1344 "locator": "",
1345 "nspace": "",
1346 "name": "ROBJ5"
1347 }
1348 },
1349 {
1350 "shards": [
1351 {
1352 "data_digest": "0x2ddbf8f5",
1353 "omap_digest": "0x689ee887",
1354 "size": 7,
1355 "errors": [
1356 "omap_digest_mismatch_oi"
1357 ],
1358 "osd": 0
1359 },
1360 {
1361 "data_digest": "0x2ddbf8f5",
1362 "omap_digest": "0x179c919f",
1363 "size": 7,
1364 "errors": [],
1365 "osd": 1
1366 }
1367 ],
1368 "selected_object_info": "2:a53c12e8:::ROBJ6:head(47'50 osd.0.0:49 dirty|omap|data_digest|omap_digest s 7 uv 18 dd 2ddbf8f5 od 179c919f alloc_hint [0 0 0])",
1369 "union_shard_errors": [
1370 "omap_digest_mismatch_oi"
1371 ],
1372 "errors": [
1373 "omap_digest_mismatch"
1374 ],
1375 "object": {
1376 "version": 18,
1377 "snap": "head",
1378 "locator": "",
1379 "nspace": "",
1380 "name": "ROBJ6"
1381 }
1382 },
1383 {
1384 "shards": [
1385 {
1386 "data_digest": "0x2ddbf8f5",
1387 "omap_digest": "0xefced57a",
1388 "size": 7,
1389 "errors": [],
1390 "osd": 0
1391 },
1392 {
1393 "data_digest": "0x2ddbf8f5",
1394 "omap_digest": "0x6a73cc07",
1395 "size": 7,
1396 "errors": [
1397 "omap_digest_mismatch_oi"
1398 ],
1399 "osd": 1
1400 }
1401 ],
1402 "selected_object_info": "2:8b55fa4b:::ROBJ7:head(47'49 osd.0.0:48 dirty|omap|data_digest|omap_digest s 7 uv 21 dd 2ddbf8f5 od efced57a alloc_hint [0 0 0])",
1403 "union_shard_errors": [
1404 "omap_digest_mismatch_oi"
1405 ],
1406 "errors": [
1407 "omap_digest_mismatch"
1408 ],
1409 "object": {
1410 "version": 21,
1411 "snap": "head",
1412 "locator": "",
1413 "nspace": "",
1414 "name": "ROBJ7"
1415 }
1416 },
1417 {
1418 "shards": [
1419 {
1420 "attrs": [
1421 {
1422 "Base64": true,
1423 "value": "",
1424 "name": "_"
1425 },
1426 {
1427 "Base64": false,
1428 "value": "bad-val",
1429 "name": "_key1-ROBJ8"
1430 },
1431 {
1432 "Base64": false,
1433 "value": "val3-ROBJ8",
1434 "name": "_key3-ROBJ8"
1435 },
1436 {
1437 "Base64": true,
1438 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1439 "name": "snapset"
1440 }
1441 ],
1442 "data_digest": "0x2ddbf8f5",
1443 "omap_digest": "0xd6be81dc",
1444 "size": 7,
1445 "errors": [],
1446 "osd": 0
1447 },
1448 {
1449 "attrs": [
1450 {
1451 "Base64": true,
1452 "value": "",
1453 "name": "_"
1454 },
1455 {
1456 "Base64": false,
1457 "value": "val1-ROBJ8",
1458 "name": "_key1-ROBJ8"
1459 },
1460 {
1461 "Base64": false,
1462 "value": "val2-ROBJ8",
1463 "name": "_key2-ROBJ8"
1464 },
1465 {
1466 "Base64": true,
1467 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1468 "name": "snapset"
1469 }
1470 ],
1471 "data_digest": "0x2ddbf8f5",
1472 "omap_digest": "0xd6be81dc",
1473 "size": 7,
1474 "errors": [],
1475 "osd": 1
1476 }
1477 ],
1478 "selected_object_info": "2:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 62 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])",
1479 "union_shard_errors": [],
1480 "errors": [
1481 "attr_value_mismatch",
1482 "attr_name_mismatch"
1483 ],
1484 "object": {
1485 "version": 62,
1486 "snap": "head",
1487 "locator": "",
1488 "nspace": "",
1489 "name": "ROBJ8"
1490 }
1491 },
1492 {
1493 "shards": [
1494 {
1495 "attrs": [
1496 {
1497 "Base64": true,
1498 "value": "",
1499 "name": "_"
1500 },
1501 {
1502 "Base64": true,
1503 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1504 "name": "snapset"
1505 }
1506 ],
1507 "object_info": "2:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])",
1508 "data_digest": "0x1f26fb26",
1509 "omap_digest": "0x2eecc539",
1510 "size": 3,
1511 "errors": [],
1512 "osd": 0
1513 },
1514 {
1515 "attrs": [
1516 {
1517 "Base64": true,
1518 "value": "",
1519 "name": "_"
1520 },
1521 {
1522 "Base64": true,
1523 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1524 "name": "snapset"
1525 }
1526 ],
1527 "object_info": "2:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 64 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])",
1528 "data_digest": "0x1f26fb26",
1529 "omap_digest": "0x2eecc539",
1530 "size": 3,
1531 "errors": [],
1532 "osd": 1
1533 }
1534 ],
1535 "selected_object_info": "2:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 64 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])",
1536 "union_shard_errors": [],
1537 "errors": [
1538 "object_info_inconsistency",
1539 "attr_value_mismatch"
1540 ],
1541 "object": {
1542 "version": 64,
1543 "snap": "head",
1544 "locator": "",
1545 "nspace": "",
1546 "name": "ROBJ9"
1547 }
1548 }
1549 ],
1550 "epoch": 0
1551 }
1552 EOF
1553
1554 jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson
1555 diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
1556 if test $getjson = "yes"
1557 then
1558 jq '.' $dir/json > save2.json
1559 fi
1560
1561 if which jsonschema > /dev/null;
1562 then
1563 jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
1564 fi
1565
1566 rados rmpool $poolname $poolname --yes-i-really-really-mean-it
1567 teardown $dir || return 1
1568 }
1569
1570
1571 #
1572 # Test scrub errors for an erasure coded pool
1573 #
1574 function corrupt_scrub_erasure() {
1575 local dir=$1
1576 local allow_overwrites=$2
1577 local poolname=ecpool
1578 local total_objs=5
1579
1580 setup $dir || return 1
1581 run_mon $dir a || return 1
1582 run_mgr $dir x || return 1
1583 for id in $(seq 0 2) ; do
1584 if [ "$allow_overwrites" = "true" ]; then
1585 run_osd_bluestore $dir $id || return 1
1586 else
1587 run_osd $dir $id || return 1
1588 fi
1589 done
1590 wait_for_clean || return 1
1591
1592 create_ec_pool $poolname $allow_overwrites k=2 m=1 stripe_unit=2K --force || return 1
1593
1594 for i in $(seq 1 $total_objs) ; do
1595 objname=EOBJ${i}
1596 add_something $dir $poolname $objname || return 1
1597
1598 local osd=$(expr $i % 2)
1599
1600 case $i in
1601 1)
1602 # Size (deep scrub data_digest too)
1603 local payload=UVWXYZZZ
1604 echo $payload > $dir/CORRUPT
1605 objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
1606 ;;
1607
1608 2)
1609 # Corrupt EC shard
1610 dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=1
1611 objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
1612 ;;
1613
1614 3)
1615 # missing
1616 objectstore_tool $dir $osd $objname remove || return 1
1617 ;;
1618
1619 4)
1620 rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
1621 rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
1622
1623 # Break xattrs
1624 echo -n bad-val > $dir/bad-val
1625 objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
1626 objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
1627 echo -n val3-$objname > $dir/newval
1628 objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
1629 rm $dir/bad-val $dir/newval
1630 ;;
1631
1632 5)
1633 # Corrupt EC shard
1634 dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=2
1635 objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
1636 ;;
1637
1638 esac
1639 done
1640
1641 local pg=$(get_pg $poolname EOBJ0)
1642
1643 pg_scrub $pg
1644
1645 rados list-inconsistent-pg $poolname > $dir/json || return 1
1646 # Check pg count
1647 test $(jq '. | length' $dir/json) = "1" || return 1
1648 # Check pgid
1649 test $(jq -r '.[0]' $dir/json) = $pg || return 1
1650
1651 rados list-inconsistent-obj $pg > $dir/json || return 1
1652 # Get epoch for repair-get requests
1653 epoch=$(jq .epoch $dir/json)
1654
1655 jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
1656 {
1657 "inconsistents": [
1658 {
1659 "shards": [
1660 {
1661 "size": 2048,
1662 "errors": [],
1663 "shard": 2,
1664 "osd": 0
1665 },
1666 {
1667 "size": 9,
1668 "errors": [
1669 "size_mismatch_oi"
1670 ],
1671 "shard": 1,
1672 "osd": 1
1673 },
1674 {
1675 "size": 2048,
1676 "errors": [],
1677 "shard": 0,
1678 "osd": 2
1679 }
1680 ],
1681 "selected_object_info": "2:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1682 "union_shard_errors": [
1683 "size_mismatch_oi"
1684 ],
1685 "errors": [
1686 "size_mismatch"
1687 ],
1688 "object": {
1689 "version": 1,
1690 "snap": "head",
1691 "locator": "",
1692 "nspace": "",
1693 "name": "EOBJ1"
1694 }
1695 },
1696 {
1697 "shards": [
1698 {
1699 "size": 2048,
1700 "errors": [],
1701 "shard": 2,
1702 "osd": 0
1703 },
1704 {
1705 "errors": [
1706 "missing"
1707 ],
1708 "shard": 1,
1709 "osd": 1
1710 },
1711 {
1712 "size": 2048,
1713 "errors": [],
1714 "shard": 0,
1715 "osd": 2
1716 }
1717 ],
1718 "selected_object_info": "2:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1719 "union_shard_errors": [
1720 "missing"
1721 ],
1722 "errors": [],
1723 "object": {
1724 "version": 3,
1725 "snap": "head",
1726 "locator": "",
1727 "nspace": "",
1728 "name": "EOBJ3"
1729 }
1730 },
1731 {
1732 "shards": [
1733 {
1734 "attrs": [
1735 {
1736 "Base64": true,
1737 "value": "",
1738 "name": "_"
1739 },
1740 {
1741 "Base64": false,
1742 "value": "bad-val",
1743 "name": "_key1-EOBJ4"
1744 },
1745 {
1746 "Base64": false,
1747 "value": "val3-EOBJ4",
1748 "name": "_key3-EOBJ4"
1749 },
1750 {
1751 "Base64": true,
1752 "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
1753 "name": "hinfo_key"
1754 },
1755 {
1756 "Base64": true,
1757 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1758 "name": "snapset"
1759 }
1760 ],
1761 "size": 2048,
1762 "errors": [],
1763 "shard": 2,
1764 "osd": 0
1765 },
1766 {
1767 "attrs": [
1768 {
1769 "Base64": true,
1770 "value": "",
1771 "name": "_"
1772 },
1773 {
1774 "Base64": false,
1775 "value": "val1-EOBJ4",
1776 "name": "_key1-EOBJ4"
1777 },
1778 {
1779 "Base64": false,
1780 "value": "val2-EOBJ4",
1781 "name": "_key2-EOBJ4"
1782 },
1783 {
1784 "Base64": true,
1785 "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
1786 "name": "hinfo_key"
1787 },
1788 {
1789 "Base64": true,
1790 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1791 "name": "snapset"
1792 }
1793 ],
1794 "size": 2048,
1795 "errors": [],
1796 "shard": 1,
1797 "osd": 1
1798 },
1799 {
1800 "attrs": [
1801 {
1802 "Base64": true,
1803 "value": "",
1804 "name": "_"
1805 },
1806 {
1807 "Base64": false,
1808 "value": "val1-EOBJ4",
1809 "name": "_key1-EOBJ4"
1810 },
1811 {
1812 "Base64": false,
1813 "value": "val2-EOBJ4",
1814 "name": "_key2-EOBJ4"
1815 },
1816 {
1817 "Base64": true,
1818 "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
1819 "name": "hinfo_key"
1820 },
1821 {
1822 "Base64": true,
1823 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1824 "name": "snapset"
1825 }
1826 ],
1827 "size": 2048,
1828 "errors": [],
1829 "shard": 0,
1830 "osd": 2
1831 }
1832 ],
1833 "selected_object_info": "2:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1834 "union_shard_errors": [],
1835 "errors": [
1836 "attr_value_mismatch",
1837 "attr_name_mismatch"
1838 ],
1839 "object": {
1840 "version": 6,
1841 "snap": "head",
1842 "locator": "",
1843 "nspace": "",
1844 "name": "EOBJ4"
1845 }
1846 },
1847 {
1848 "shards": [
1849 {
1850 "size": 2048,
1851 "errors": [],
1852 "shard": 2,
1853 "osd": 0
1854 },
1855 {
1856 "size": 4096,
1857 "errors": [
1858 "size_mismatch_oi"
1859 ],
1860 "shard": 1,
1861 "osd": 1
1862 },
1863 {
1864 "size": 2048,
1865 "errors": [],
1866 "shard": 0,
1867 "osd": 2
1868 }
1869 ],
1870 "selected_object_info": "2:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1871 "union_shard_errors": [
1872 "size_mismatch_oi"
1873 ],
1874 "errors": [
1875 "size_mismatch"
1876 ],
1877 "object": {
1878 "version": 7,
1879 "snap": "head",
1880 "locator": "",
1881 "nspace": "",
1882 "name": "EOBJ5"
1883 }
1884 }
1885 ],
1886 "epoch": 0
1887 }
1888 EOF
1889
1890 jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson
1891 diff -y $termwidth $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
1892 if test $getjson = "yes"
1893 then
1894 jq '.' $dir/json > save3.json
1895 fi
1896
1897 if which jsonschema > /dev/null;
1898 then
1899 jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
1900 fi
1901
1902 pg_deep_scrub $pg
1903
1904 rados list-inconsistent-pg $poolname > $dir/json || return 1
1905 # Check pg count
1906 test $(jq '. | length' $dir/json) = "1" || return 1
1907 # Check pgid
1908 test $(jq -r '.[0]' $dir/json) = $pg || return 1
1909
1910 rados list-inconsistent-obj $pg > $dir/json || return 1
1911 # Get epoch for repair-get requests
1912 epoch=$(jq .epoch $dir/json)
1913
1914 if [ "$allow_overwrites" = "true" ]
1915 then
1916 jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
1917 {
1918 "inconsistents": [
1919 {
1920 "shards": [
1921 {
1922 "data_digest": "0x00000000",
1923 "omap_digest": "0xffffffff",
1924 "size": 2048,
1925 "errors": [],
1926 "shard": 2,
1927 "osd": 0
1928 },
1929 {
1930 "size": 9,
1931 "errors": [
1932 "read_error",
1933 "size_mismatch_oi"
1934 ],
1935 "shard": 1,
1936 "osd": 1
1937 },
1938 {
1939 "data_digest": "0x00000000",
1940 "omap_digest": "0xffffffff",
1941 "size": 2048,
1942 "errors": [],
1943 "shard": 0,
1944 "osd": 2
1945 }
1946 ],
1947 "selected_object_info": "2:9175b684:::EOBJ1:head(27'1 client.4155.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1948 "union_shard_errors": [
1949 "read_error",
1950 "size_mismatch_oi"
1951 ],
1952 "errors": [
1953 "size_mismatch"
1954 ],
1955 "object": {
1956 "version": 1,
1957 "snap": "head",
1958 "locator": "",
1959 "nspace": "",
1960 "name": "EOBJ1"
1961 }
1962 },
1963 {
1964 "shards": [
1965 {
1966 "data_digest": "0x00000000",
1967 "omap_digest": "0xffffffff",
1968 "size": 2048,
1969 "errors": [],
1970 "shard": 2,
1971 "osd": 0
1972 },
1973 {
1974 "errors": [
1975 "missing"
1976 ],
1977 "shard": 1,
1978 "osd": 1
1979 },
1980 {
1981 "data_digest": "0x00000000",
1982 "omap_digest": "0xffffffff",
1983 "size": 2048,
1984 "errors": [],
1985 "shard": 0,
1986 "osd": 2
1987 }
1988 ],
1989 "selected_object_info": "2:b197b25d:::EOBJ3:head(41'3 client.4199.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1990 "union_shard_errors": [
1991 "missing"
1992 ],
1993 "errors": [],
1994 "object": {
1995 "version": 3,
1996 "snap": "head",
1997 "locator": "",
1998 "nspace": "",
1999 "name": "EOBJ3"
2000 }
2001 },
2002 {
2003 "shards": [
2004 {
2005 "attrs": [
2006 {
2007 "Base64": true,
2008 "value": "",
2009 "name": "_"
2010 },
2011 {
2012 "Base64": false,
2013 "value": "bad-val",
2014 "name": "_key1-EOBJ4"
2015 },
2016 {
2017 "Base64": false,
2018 "value": "val3-EOBJ4",
2019 "name": "_key3-EOBJ4"
2020 },
2021 {
2022 "Base64": true,
2023 "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2024 "name": "hinfo_key"
2025 },
2026 {
2027 "Base64": true,
2028 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2029 "name": "snapset"
2030 }
2031 ],
2032 "data_digest": "0x00000000",
2033 "omap_digest": "0xffffffff",
2034 "size": 2048,
2035 "errors": [],
2036 "shard": 2,
2037 "osd": 0
2038 },
2039 {
2040 "attrs": [
2041 {
2042 "Base64": true,
2043 "value": "",
2044 "name": "_"
2045 },
2046 {
2047 "Base64": false,
2048 "value": "val1-EOBJ4",
2049 "name": "_key1-EOBJ4"
2050 },
2051 {
2052 "Base64": false,
2053 "value": "val2-EOBJ4",
2054 "name": "_key2-EOBJ4"
2055 },
2056 {
2057 "Base64": true,
2058 "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2059 "name": "hinfo_key"
2060 },
2061 {
2062 "Base64": true,
2063 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2064 "name": "snapset"
2065 }
2066 ],
2067 "data_digest": "0x00000000",
2068 "omap_digest": "0xffffffff",
2069 "size": 2048,
2070 "errors": [],
2071 "shard": 1,
2072 "osd": 1
2073 },
2074 {
2075 "attrs": [
2076 {
2077 "Base64": true,
2078 "value": "",
2079 "name": "_"
2080 },
2081 {
2082 "Base64": false,
2083 "value": "val1-EOBJ4",
2084 "name": "_key1-EOBJ4"
2085 },
2086 {
2087 "Base64": false,
2088 "value": "val2-EOBJ4",
2089 "name": "_key2-EOBJ4"
2090 },
2091 {
2092 "Base64": true,
2093 "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2094 "name": "hinfo_key"
2095 },
2096 {
2097 "Base64": true,
2098 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2099 "name": "snapset"
2100 }
2101 ],
2102 "data_digest": "0x00000000",
2103 "omap_digest": "0xffffffff",
2104 "size": 2048,
2105 "errors": [],
2106 "shard": 0,
2107 "osd": 2
2108 }
2109 ],
2110 "selected_object_info": "2:5e723e06:::EOBJ4:head(48'6 client.4223.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2111 "union_shard_errors": [],
2112 "errors": [
2113 "attr_value_mismatch",
2114 "attr_name_mismatch"
2115 ],
2116 "object": {
2117 "version": 6,
2118 "snap": "head",
2119 "locator": "",
2120 "nspace": "",
2121 "name": "EOBJ4"
2122 }
2123 },
2124 {
2125 "shards": [
2126 {
2127 "data_digest": "0x00000000",
2128 "omap_digest": "0xffffffff",
2129 "size": 2048,
2130 "errors": [],
2131 "shard": 2,
2132 "osd": 0
2133 },
2134 {
2135 "data_digest": "0x00000000",
2136 "omap_digest": "0xffffffff",
2137 "size": 4096,
2138 "errors": [
2139 "size_mismatch_oi"
2140 ],
2141 "shard": 1,
2142 "osd": 1
2143 },
2144 {
2145 "data_digest": "0x00000000",
2146 "omap_digest": "0xffffffff",
2147 "size": 2048,
2148 "errors": [],
2149 "shard": 0,
2150 "osd": 2
2151 }
2152 ],
2153 "selected_object_info": "2:8549dfb5:::EOBJ5:head(65'7 client.4288.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2154 "union_shard_errors": [
2155 "size_mismatch_oi"
2156 ],
2157 "errors": [
2158 "size_mismatch"
2159 ],
2160 "object": {
2161 "version": 7,
2162 "snap": "head",
2163 "locator": "",
2164 "nspace": "",
2165 "name": "EOBJ5"
2166 }
2167 }
2168 ],
2169 "epoch": 0
2170 }
2171 EOF
2172
2173 else
2174
2175 jq "$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
2176 {
2177 "inconsistents": [
2178 {
2179 "shards": [
2180 {
2181 "data_digest": "0x04cfa72f",
2182 "omap_digest": "0xffffffff",
2183 "size": 2048,
2184 "errors": [],
2185 "shard": 2,
2186 "osd": 0
2187 },
2188 {
2189 "size": 9,
2190 "errors": [
2191 "read_error",
2192 "size_mismatch_oi"
2193 ],
2194 "shard": 1,
2195 "osd": 1
2196 },
2197 {
2198 "data_digest": "0x04cfa72f",
2199 "omap_digest": "0xffffffff",
2200 "size": 2048,
2201 "errors": [],
2202 "shard": 0,
2203 "osd": 2
2204 }
2205 ],
2206 "selected_object_info": "2:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2207 "union_shard_errors": [
2208 "read_error",
2209 "size_mismatch_oi"
2210 ],
2211 "errors": [
2212 "size_mismatch"
2213 ],
2214 "object": {
2215 "version": 1,
2216 "snap": "head",
2217 "locator": "",
2218 "nspace": "",
2219 "name": "EOBJ1"
2220 }
2221 },
2222 {
2223 "shards": [
2224 {
2225 "size": 2048,
2226 "errors": [
2227 "ec_hash_error"
2228 ],
2229 "shard": 2,
2230 "osd": 0
2231 },
2232 {
2233 "data_digest": "0x04cfa72f",
2234 "omap_digest": "0xffffffff",
2235 "size": 2048,
2236 "errors": [],
2237 "shard": 1,
2238 "osd": 1
2239 },
2240 {
2241 "data_digest": "0x04cfa72f",
2242 "omap_digest": "0xffffffff",
2243 "size": 2048,
2244 "errors": [],
2245 "shard": 0,
2246 "osd": 2
2247 }
2248 ],
2249 "selected_object_info": "2:9babd184:::EOBJ2:head(29'2 client.4217.0:1 dirty|data_digest|omap_digest s 7 uv 2 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2250 "union_shard_errors": [
2251 "ec_hash_error"
2252 ],
2253 "errors": [],
2254 "object": {
2255 "version": 2,
2256 "snap": "head",
2257 "locator": "",
2258 "nspace": "",
2259 "name": "EOBJ2"
2260 }
2261 },
2262 {
2263 "shards": [
2264 {
2265 "data_digest": "0x04cfa72f",
2266 "omap_digest": "0xffffffff",
2267 "size": 2048,
2268 "errors": [],
2269 "shard": 2,
2270 "osd": 0
2271 },
2272 {
2273 "errors": [
2274 "missing"
2275 ],
2276 "shard": 1,
2277 "osd": 1
2278 },
2279 {
2280 "data_digest": "0x04cfa72f",
2281 "omap_digest": "0xffffffff",
2282 "size": 2048,
2283 "errors": [],
2284 "shard": 0,
2285 "osd": 2
2286 }
2287 ],
2288 "selected_object_info": "2:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2289 "union_shard_errors": [
2290 "missing"
2291 ],
2292 "errors": [],
2293 "object": {
2294 "version": 3,
2295 "snap": "head",
2296 "locator": "",
2297 "nspace": "",
2298 "name": "EOBJ3"
2299 }
2300 },
2301 {
2302 "shards": [
2303 {
2304 "attrs": [
2305 {
2306 "Base64": true,
2307 "value": "",
2308 "name": "_"
2309 },
2310 {
2311 "Base64": false,
2312 "value": "bad-val",
2313 "name": "_key1-EOBJ4"
2314 },
2315 {
2316 "Base64": false,
2317 "value": "val3-EOBJ4",
2318 "name": "_key3-EOBJ4"
2319 },
2320 {
2321 "Base64": true,
2322 "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2323 "name": "hinfo_key"
2324 },
2325 {
2326 "Base64": true,
2327 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2328 "name": "snapset"
2329 }
2330 ],
2331 "data_digest": "0x04cfa72f",
2332 "omap_digest": "0xffffffff",
2333 "size": 2048,
2334 "errors": [],
2335 "shard": 2,
2336 "osd": 0
2337 },
2338 {
2339 "attrs": [
2340 {
2341 "Base64": true,
2342 "value": "",
2343 "name": "_"
2344 },
2345 {
2346 "Base64": false,
2347 "value": "val1-EOBJ4",
2348 "name": "_key1-EOBJ4"
2349 },
2350 {
2351 "Base64": false,
2352 "value": "val2-EOBJ4",
2353 "name": "_key2-EOBJ4"
2354 },
2355 {
2356 "Base64": true,
2357 "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2358 "name": "hinfo_key"
2359 },
2360 {
2361 "Base64": true,
2362 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2363 "name": "snapset"
2364 }
2365 ],
2366 "data_digest": "0x04cfa72f",
2367 "omap_digest": "0xffffffff",
2368 "size": 2048,
2369 "errors": [],
2370 "shard": 1,
2371 "osd": 1
2372 },
2373 {
2374 "attrs": [
2375 {
2376 "Base64": true,
2377 "value": "",
2378 "name": "_"
2379 },
2380 {
2381 "Base64": false,
2382 "value": "val1-EOBJ4",
2383 "name": "_key1-EOBJ4"
2384 },
2385 {
2386 "Base64": false,
2387 "value": "val2-EOBJ4",
2388 "name": "_key2-EOBJ4"
2389 },
2390 {
2391 "Base64": true,
2392 "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2393 "name": "hinfo_key"
2394 },
2395 {
2396 "Base64": true,
2397 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2398 "name": "snapset"
2399 }
2400 ],
2401 "data_digest": "0x04cfa72f",
2402 "omap_digest": "0xffffffff",
2403 "size": 2048,
2404 "errors": [],
2405 "shard": 0,
2406 "osd": 2
2407 }
2408 ],
2409 "selected_object_info": "2:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2410 "union_shard_errors": [],
2411 "errors": [
2412 "attr_value_mismatch",
2413 "attr_name_mismatch"
2414 ],
2415 "object": {
2416 "version": 6,
2417 "snap": "head",
2418 "locator": "",
2419 "nspace": "",
2420 "name": "EOBJ4"
2421 }
2422 },
2423 {
2424 "shards": [
2425 {
2426 "data_digest": "0x04cfa72f",
2427 "omap_digest": "0xffffffff",
2428 "size": 2048,
2429 "errors": [],
2430 "shard": 2,
2431 "osd": 0
2432 },
2433 {
2434 "size": 4096,
2435 "errors": [
2436 "size_mismatch_oi",
2437 "ec_size_error"
2438 ],
2439 "shard": 1,
2440 "osd": 1
2441 },
2442 {
2443 "data_digest": "0x04cfa72f",
2444 "omap_digest": "0xffffffff",
2445 "size": 2048,
2446 "errors": [],
2447 "shard": 0,
2448 "osd": 2
2449 }
2450 ],
2451 "selected_object_info": "2:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2452 "union_shard_errors": [
2453 "size_mismatch_oi",
2454 "ec_size_error"
2455 ],
2456 "errors": [
2457 "size_mismatch"
2458 ],
2459 "object": {
2460 "version": 7,
2461 "snap": "head",
2462 "locator": "",
2463 "nspace": "",
2464 "name": "EOBJ5"
2465 }
2466 }
2467 ],
2468 "epoch": 0
2469 }
2470 EOF
2471
2472 fi
2473
2474 jq "$jqfilter" $dir/json | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/csjson
2475 diff -y $termwidth $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
2476 if test $getjson = "yes"
2477 then
2478 if [ "$allow_overwrites" = "true" ]
2479 then
2480 num=4
2481 else
2482 num=5
2483 fi
2484 jq '.' $dir/json > save${num}.json
2485 fi
2486
2487 if which jsonschema > /dev/null;
2488 then
2489 jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
2490 fi
2491
2492 rados rmpool $poolname $poolname --yes-i-really-really-mean-it
2493 teardown $dir || return 1
2494 }
2495
2496 function TEST_corrupt_scrub_erasure_appends() {
2497 corrupt_scrub_erasure $1 false
2498 }
2499
2500 function TEST_corrupt_scrub_erasure_overwrites() {
2501 corrupt_scrub_erasure $1 true
2502 }
2503
2504 #
2505 # Test to make sure that a periodic scrub won't cause deep-scrub info to be lost
2506 #
2507 function TEST_periodic_scrub_replicated() {
2508 local dir=$1
2509 local poolname=psr_pool
2510 local objname=POBJ
2511
2512 setup $dir || return 1
2513 run_mon $dir a --osd_pool_default_size=2 || return 1
2514 run_mgr $dir x || return 1
2515 local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0"
2516 run_osd $dir 0 $ceph_osd_args || return 1
2517 run_osd $dir 1 $ceph_osd_args || return 1
2518 wait_for_clean || return 1
2519
2520 ceph osd pool create $poolname 1 1 || return 1
2521 wait_for_clean || return 1
2522
2523 local osd=0
2524 add_something $dir $poolname $objname scrub || return 1
2525 local primary=$(get_primary $poolname $objname)
2526 local pg=$(get_pg $poolname $objname)
2527
2528 # Add deep-scrub only error
2529 local payload=UVWXYZ
2530 echo $payload > $dir/CORRUPT
2531 # Uses $ceph_osd_args for osd restart
2532 objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
2533
2534 # No scrub information available, so expect failure
2535 set -o pipefail
2536 ! rados list-inconsistent-obj $pg | jq '.' || return 1
2537 set +o pipefail
2538
2539 pg_deep_scrub $pg || return 1
2540
2541 # Make sure bad object found
2542 rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
2543
2544 local last_scrub=$(get_last_scrub_stamp $pg)
2545 # Fake a schedule scrub
2546 CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.${primary}.asok \
2547 trigger_scrub $pg || return 1
2548 # Wait for schedule regular scrub
2549 wait_for_scrub $pg "$last_scrub"
2550
2551 # It needed to be upgraded
2552 grep -q "Deep scrub errors, upgrading scrub to deep-scrub" $dir/osd.${primary}.log || return 1
2553
2554 # Bad object still known
2555 rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
2556
2557 # Can't upgrade with this set
2558 ceph osd set nodeep-scrub
2559
2560 # Fake a schedule scrub
2561 local last_scrub=$(get_last_scrub_stamp $pg)
2562 CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.${primary}.asok \
2563 trigger_scrub $pg || return 1
2564 # Wait for schedule regular scrub
2565 # to notice scrub and skip it
2566 local found=false
2567 for i in $(seq 14 -1 0)
2568 do
2569 sleep 1
2570 ! grep -q "Regular scrub skipped due to deep-scrub errors and nodeep-scrub set" $dir/osd.${primary}.log || { found=true ; break; }
2571 echo Time left: $i seconds
2572 done
2573 test $found = "true" || return 1
2574
2575 # Bad object still known
2576 rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
2577
2578 # Request a regular scrub and it will be done
2579 local scrub_backoff_ratio=$(get_config osd ${primary} osd_scrub_backoff_ratio)
2580 set_config osd ${primary} osd_scrub_backoff_ratio 0
2581 pg_scrub $pg
2582 sleep 1
2583 set_config osd ${primary} osd_scrub_backoff_ratio $scrub_backoff_ratio
2584 grep -q "Regular scrub request, losing deep-scrub details" $dir/osd.${primary}.log || return 1
2585
2586 # deep-scrub error is no longer present
2587 rados list-inconsistent-obj $pg | jq '.' | grep -qv $objname || return 1
2588 }
2589
2590
2591 main osd-scrub-repair "$@"
2592
2593 # Local Variables:
2594 # compile-command: "cd ../.. ; make -j4 && \
2595 # test/osd/osd-scrub-repair.sh # TEST_corrupt_and_repair_replicated"
2596 # End:
Ceph