3 # Copyright (C) 2014 Red Hat <contact@redhat.com>
5 # Author: Loic Dachary <loic@dachary.org>
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU Library Public License as published by
9 # the Free Software Foundation; either version 2, or (at your option)
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Library Public License for more details.
17 source $CEPH_ROOT/qa
/standalone
/ceph-helpers.sh
19 if [ `uname` = FreeBSD
]; then
20 # erasure coding overwrites are only tested on Bluestore
21 # erasure coding on filestore is unsafe
22 # http://docs.ceph.com/docs/master/rados/operations/erasure-code/#erasure-coding-with-overwrites
23 use_ec_overwrite
=false
28 # Test development and debugging
29 # Set to "yes" in order to ignore diff errors and save results to update test
32 # Ignore the epoch and filter out the attr '_' value because it has date information and won't match
33 if [ "$(jq --version 2>&1 | awk '{ print $3}')" = "1.3" ]; then # Not sure all versions that apply here
34 jqfilter
='.inconsistents | (.[].shards[].attrs[] | select(.name == "_") | .value) |= "----Stripped-by-test----"'
36 jqfilter
='.inconsistents | (.[].shards[].attrs[]? | select(.name == "_") | .value) |= "----Stripped-by-test----"'
38 sortkeys
='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)'
40 # Remove items are not consistent across runs, the pg interval and client
41 sedfilter
='s/\([ ]*\"\(selected_\)*object_info\":.*head[(]\)[^[:space:]]* [^[:space:]]* \(.*\)/\1\3/'
47 export CEPH_MON
="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one
49 CEPH_ARGS
+="--fsid=$(uuidgen) --auth-supported=none "
50 CEPH_ARGS
+="--mon-host=$CEPH_MON "
52 local funcs
=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
53 for func
in $funcs ; do
54 $func $dir ||
return 1
58 function add_something
() {
61 local obj
=${3:-SOMETHING}
62 local scrub
=${4:-noscrub}
64 if [ "$scrub" = "noscrub" ];
66 ceph osd
set noscrub ||
return 1
67 ceph osd
set nodeep-scrub ||
return 1
69 ceph osd
unset noscrub ||
return 1
70 ceph osd
unset nodeep-scrub ||
return 1
74 echo $payload > $dir/ORIGINAL
75 rados
--pool $poolname put
$obj $dir/ORIGINAL ||
return 1
79 # Corrupt one copy of a replicated pool
81 function TEST_corrupt_and_repair_replicated
() {
85 setup
$dir ||
return 1
86 run_mon
$dir a
--osd_pool_default_size=2 ||
return 1
87 run_mgr
$dir x ||
return 1
88 run_osd
$dir 0 ||
return 1
89 run_osd
$dir 1 ||
return 1
90 create_rbd_pool ||
return 1
91 wait_for_clean ||
return 1
93 add_something
$dir $poolname ||
return 1
94 corrupt_and_repair_one
$dir $poolname $
(get_not_primary
$poolname SOMETHING
) ||
return 1
95 # Reproduces http://tracker.ceph.com/issues/8914
96 corrupt_and_repair_one
$dir $poolname $
(get_primary
$poolname SOMETHING
) ||
return 1
98 teardown
$dir ||
return 1
101 function corrupt_and_repair_two
() {
108 # 1) remove the corresponding file from the OSDs
111 run_in_background pids objectstore_tool
$dir $first SOMETHING remove
112 run_in_background pids objectstore_tool
$dir $second SOMETHING remove
115 if [ $return_code -ne 0 ]; then return $return_code; fi
120 local pg
=$
(get_pg
$poolname SOMETHING
)
123 # 3) The files must be back
126 run_in_background pids objectstore_tool
$dir $first SOMETHING list-attrs
127 run_in_background pids objectstore_tool
$dir $second SOMETHING list-attrs
130 if [ $return_code -ne 0 ]; then return $return_code; fi
132 rados
--pool $poolname get SOMETHING
$dir/COPY ||
return 1
133 diff $dir/ORIGINAL
$dir/COPY ||
return 1
138 # 2) remove the corresponding file from a designated OSD
140 # 4) check that the file has been restored in the designated OSD
142 function corrupt_and_repair_one
() {
148 # 1) remove the corresponding file from the OSD
150 objectstore_tool
$dir $osd SOMETHING remove ||
return 1
154 local pg
=$
(get_pg
$poolname SOMETHING
)
157 # 3) The file must be back
159 objectstore_tool
$dir $osd SOMETHING list-attrs ||
return 1
160 rados
--pool $poolname get SOMETHING
$dir/COPY ||
return 1
161 diff $dir/ORIGINAL
$dir/COPY ||
return 1
164 function corrupt_and_repair_erasure_coded
() {
168 add_something
$dir $poolname ||
return 1
170 local primary
=$
(get_primary
$poolname SOMETHING
)
171 local -a osds
=($
(get_osds
$poolname SOMETHING |
sed -e "s/$primary//"))
172 local not_primary_first
=${osds[0]}
173 local not_primary_second
=${osds[1]}
175 # Reproduces http://tracker.ceph.com/issues/10017
176 corrupt_and_repair_one
$dir $poolname $primary ||
return 1
177 # Reproduces http://tracker.ceph.com/issues/10409
178 corrupt_and_repair_one
$dir $poolname $not_primary_first ||
return 1
179 corrupt_and_repair_two
$dir $poolname $not_primary_first $not_primary_second ||
return 1
180 corrupt_and_repair_two
$dir $poolname $primary $not_primary_first ||
return 1
184 function create_ec_pool
() {
186 local allow_overwrites
=$2
188 ceph osd erasure-code-profile
set myprofile crush-failure-domain
=osd
$3 $4 $5 $6 $7 ||
return 1
190 create_pool
"$poolname" 1 1 erasure myprofile ||
return 1
192 if [ "$allow_overwrites" = "true" ]; then
193 ceph osd pool
set "$poolname" allow_ec_overwrites true ||
return 1
196 wait_for_clean ||
return 1
200 function auto_repair_erasure_coded
() {
202 local allow_overwrites
=$2
203 local poolname
=ecpool
205 # Launch a cluster with 5 seconds scrub interval
206 setup
$dir ||
return 1
207 run_mon
$dir a ||
return 1
208 run_mgr
$dir x ||
return 1
209 local ceph_osd_args
="--osd-scrub-auto-repair=true \
210 --osd-deep-scrub-interval=5 \
211 --osd-scrub-max-interval=5 \
212 --osd-scrub-min-interval=5 \
213 --osd-scrub-interval-randomize-ratio=0"
214 for id
in $
(seq 0 2) ; do
215 if [ "$allow_overwrites" = "true" ]; then
216 run_osd_bluestore
$dir $id $ceph_osd_args ||
return 1
218 run_osd
$dir $id $ceph_osd_args ||
return 1
221 create_rbd_pool ||
return 1
222 wait_for_clean ||
return 1
225 create_ec_pool
$poolname $allow_overwrites k
=2 m
=1 ||
return 1
229 echo $payload > $dir/ORIGINAL
230 rados
--pool $poolname put SOMETHING
$dir/ORIGINAL ||
return 1
232 # Remove the object from one shard physically
233 # Restarted osd get $ceph_osd_args passed
234 objectstore_tool
$dir $
(get_not_primary
$poolname SOMETHING
) SOMETHING remove ||
return 1
235 # Wait for auto repair
236 local pgid
=$
(get_pg
$poolname SOMETHING
)
237 wait_for_scrub
$pgid "$(get_last_scrub_stamp $pgid)"
238 wait_for_clean ||
return 1
239 # Verify - the file should be back
240 # Restarted osd get $ceph_osd_args passed
241 objectstore_tool
$dir $
(get_not_primary
$poolname SOMETHING
) SOMETHING list-attrs ||
return 1
242 rados
--pool $poolname get SOMETHING
$dir/COPY ||
return 1
243 diff $dir/ORIGINAL
$dir/COPY ||
return 1
246 teardown
$dir ||
return 1
249 function TEST_auto_repair_erasure_coded_appends
() {
250 auto_repair_erasure_coded
$1 false
253 function TEST_auto_repair_erasure_coded_overwrites
() {
254 if [ "$use_ec_overwrite" = "true" ]; then
255 auto_repair_erasure_coded
$1 true
259 function corrupt_and_repair_jerasure
() {
261 local allow_overwrites
=$2
262 local poolname
=ecpool
264 setup
$dir ||
return 1
265 run_mon
$dir a ||
return 1
266 run_mgr
$dir x ||
return 1
267 for id
in $
(seq 0 3) ; do
268 if [ "$allow_overwrites" = "true" ]; then
269 run_osd_bluestore
$dir $id ||
return 1
271 run_osd
$dir $id ||
return 1
274 create_rbd_pool ||
return 1
275 wait_for_clean ||
return 1
277 create_ec_pool
$poolname $allow_overwrites k
=2 m
=2 ||
return 1
278 corrupt_and_repair_erasure_coded
$dir $poolname ||
return 1
280 teardown
$dir ||
return 1
283 function TEST_corrupt_and_repair_jerasure_appends
() {
284 corrupt_and_repair_jerasure
$1
287 function TEST_corrupt_and_repair_jerasure_overwrites
() {
288 if [ "$use_ec_overwrite" = "true" ]; then
289 corrupt_and_repair_jerasure
$1 true
293 function corrupt_and_repair_lrc
() {
295 local allow_overwrites
=$2
296 local poolname
=ecpool
298 setup
$dir ||
return 1
299 run_mon
$dir a ||
return 1
300 run_mgr
$dir x ||
return 1
301 for id
in $
(seq 0 9) ; do
302 if [ "$allow_overwrites" = "true" ]; then
303 run_osd_bluestore
$dir $id ||
return 1
305 run_osd
$dir $id ||
return 1
308 create_rbd_pool ||
return 1
309 wait_for_clean ||
return 1
311 create_ec_pool
$poolname $allow_overwrites k
=4 m
=2 l
=3 plugin
=lrc ||
return 1
312 corrupt_and_repair_erasure_coded
$dir $poolname ||
return 1
314 teardown
$dir ||
return 1
317 function TEST_corrupt_and_repair_lrc_appends
() {
318 corrupt_and_repair_jerasure
$1
321 function TEST_corrupt_and_repair_lrc_overwrites
() {
322 if [ "$use_ec_overwrite" = "true" ]; then
323 corrupt_and_repair_jerasure
$1 true
327 function unfound_erasure_coded
() {
329 local allow_overwrites
=$2
330 local poolname
=ecpool
333 setup
$dir ||
return 1
334 run_mon
$dir a ||
return 1
335 run_mgr
$dir x ||
return 1
336 for id
in $
(seq 0 3) ; do
337 if [ "$allow_overwrites" = "true" ]; then
338 run_osd_bluestore
$dir $id ||
return 1
340 run_osd
$dir $id ||
return 1
343 create_rbd_pool ||
return 1
344 wait_for_clean ||
return 1
346 create_ec_pool
$poolname $allow_overwrites k
=2 m
=2 ||
return 1
348 add_something
$dir $poolname ||
return 1
350 local primary
=$
(get_primary
$poolname SOMETHING
)
351 local -a osds
=($
(get_osds
$poolname SOMETHING |
sed -e "s/$primary//"))
352 local not_primary_first
=${osds[0]}
353 local not_primary_second
=${osds[1]}
354 local not_primary_third
=${osds[2]}
357 # 1) remove the corresponding file from the OSDs
360 run_in_background pids objectstore_tool
$dir $not_primary_first SOMETHING remove
361 run_in_background pids objectstore_tool
$dir $not_primary_second SOMETHING remove
362 run_in_background pids objectstore_tool
$dir $not_primary_third SOMETHING remove
365 if [ $return_code -ne 0 ]; then return $return_code; fi
370 local pg
=$
(get_pg
$poolname SOMETHING
)
375 # it may take a bit to appear due to mon/mgr asynchrony
376 for f
in `seq 1 60`; do
377 ceph
-s |
grep "1/1 objects unfound" && break
380 ceph
-s|
grep "4 osds: 4 up, 4 in" ||
return 1
381 ceph
-s|
grep "1/1 objects unfound" ||
return 1
383 teardown
$dir ||
return 1
386 function TEST_unfound_erasure_coded_appends
() {
387 unfound_erasure_coded
$1
390 function TEST_unfound_erasure_coded_overwrites
() {
391 if [ "$use_ec_overwrite" = "true" ]; then
392 unfound_erasure_coded
$1 true
397 # list_missing for EC pool
399 function list_missing_erasure_coded
() {
401 local allow_overwrites
=$2
402 local poolname
=ecpool
404 setup
$dir ||
return 1
405 run_mon
$dir a ||
return 1
406 run_mgr
$dir x ||
return 1
407 for id
in $
(seq 0 2) ; do
408 if [ "$allow_overwrites" = "true" ]; then
409 run_osd_bluestore
$dir $id ||
return 1
411 run_osd
$dir $id ||
return 1
414 create_rbd_pool ||
return 1
415 wait_for_clean ||
return 1
417 create_ec_pool
$poolname $allow_overwrites k
=2 m
=1 ||
return 1
419 # Put an object and remove the two shards (including primary)
420 add_something
$dir $poolname MOBJ0 ||
return 1
421 local -a osds0
=($
(get_osds
$poolname MOBJ0
))
423 # Put another object and remove two shards (excluding primary)
424 add_something
$dir $poolname MOBJ1 ||
return 1
425 local -a osds1
=($
(get_osds
$poolname MOBJ1
))
427 # Stop all osd daemons
428 for id
in $
(seq 0 2) ; do
429 kill_daemons
$dir TERM osd.
$id >&2 < /dev
/null ||
return 1
433 ceph-objectstore-tool
--data-path $dir/$id \
434 MOBJ0 remove ||
return 1
436 ceph-objectstore-tool
--data-path $dir/$id \
437 MOBJ0 remove ||
return 1
440 ceph-objectstore-tool
--data-path $dir/$id \
441 MOBJ1 remove ||
return 1
443 ceph-objectstore-tool
--data-path $dir/$id \
444 MOBJ1 remove ||
return 1
446 for id
in $
(seq 0 2) ; do
447 activate_osd
$dir $id >&2 ||
return 1
449 create_rbd_pool ||
return 1
450 wait_for_clean ||
return 1
452 # Get get - both objects should in the same PG
453 local pg
=$
(get_pg
$poolname MOBJ0
)
455 # Repair the PG, which triggers the recovering,
456 # and should mark the object as unfound
459 for i
in $
(seq 0 120) ; do
460 [ $i -lt 60 ] ||
return 1
461 matches
=$
(ceph pg
$pg list_missing |
egrep "MOBJ0|MOBJ1" |
wc -l)
462 [ $matches -eq 2 ] && break
465 teardown
$dir ||
return 1
468 function TEST_list_missing_erasure_coded_appends
() {
469 list_missing_erasure_coded
$1 false
472 function TEST_list_missing_erasure_coded_overwrites
() {
473 if [ "$use_ec_overwrite" = "true" ]; then
474 list_missing_erasure_coded
$1 true
479 # Corrupt one copy of a replicated pool
481 function TEST_corrupt_scrub_replicated
() {
483 local poolname
=csr_pool
486 setup
$dir ||
return 1
487 run_mon
$dir a
--osd_pool_default_size=2 ||
return 1
488 run_mgr
$dir x ||
return 1
489 run_osd
$dir 0 ||
return 1
490 run_osd
$dir 1 ||
return 1
491 create_rbd_pool ||
return 1
492 wait_for_clean ||
return 1
494 create_pool foo
1 ||
return 1
495 create_pool
$poolname 1 1 ||
return 1
496 wait_for_clean ||
return 1
498 for i
in $
(seq 1 $total_objs) ; do
500 add_something
$dir $poolname $objname ||
return 1
502 rados
--pool $poolname setomapheader
$objname hdr-
$objname ||
return 1
503 rados
--pool $poolname setomapval
$objname key-
$objname val-
$objname ||
return 1
506 local pg
=$
(get_pg
$poolname ROBJ0
)
508 # Compute an old omap digest and save oi
509 CEPH_ARGS
='' ceph daemon $
(get_asok_path osd
.0) \
510 config
set osd_deep_scrub_update_digest_min_age
0
511 CEPH_ARGS
='' ceph daemon $
(get_asok_path osd
.1) \
512 config
set osd_deep_scrub_update_digest_min_age
0
515 for i
in $
(seq 1 $total_objs) ; do
518 # Alternate corruption between osd.0 and osd.1
519 local osd
=$
(expr $i % 2)
523 # Size (deep scrub data_digest too)
524 local payload
=UVWXYZZZ
525 echo $payload > $dir/CORRUPT
526 objectstore_tool
$dir $osd $objname set-bytes
$dir/CORRUPT ||
return 1
530 # digest (deep scrub only)
532 echo $payload > $dir/CORRUPT
533 objectstore_tool
$dir $osd $objname set-bytes
$dir/CORRUPT ||
return 1
538 objectstore_tool
$dir $osd $objname remove ||
return 1
542 # Modify omap value (deep scrub only)
543 objectstore_tool
$dir $osd $objname set-omap key-
$objname $dir/CORRUPT ||
return 1
547 # Delete omap key (deep scrub only)
548 objectstore_tool
$dir $osd $objname rm-omap key-
$objname ||
return 1
552 # Add extra omap key (deep scrub only)
553 echo extra
> $dir/extra-val
554 objectstore_tool
$dir $osd $objname set-omap key2-
$objname $dir/extra-val ||
return 1
559 # Modify omap header (deep scrub only)
560 echo -n newheader
> $dir/hdr
561 objectstore_tool
$dir $osd $objname set-omaphdr
$dir/hdr ||
return 1
566 rados
--pool $poolname setxattr
$objname key1-
$objname val1-
$objname ||
return 1
567 rados
--pool $poolname setxattr
$objname key2-
$objname val2-
$objname ||
return 1
570 echo -n bad-val
> $dir/bad-val
571 objectstore_tool
$dir $osd $objname set-attr _key1-
$objname $dir/bad-val ||
return 1
572 objectstore_tool
$dir $osd $objname rm-attr _key2-
$objname ||
return 1
573 echo -n val3-
$objname > $dir/newval
574 objectstore_tool
$dir $osd $objname set-attr _key3-
$objname $dir/newval ||
return 1
575 rm $dir/bad-val
$dir/newval
579 objectstore_tool
$dir $osd $objname get-attr _
> $dir/robj9-oi
580 echo -n D
> $dir/change
581 rados
--pool $poolname put
$objname $dir/change
582 objectstore_tool
$dir $osd $objname set-attr _
$dir/robj9-oi
583 rm $dir/oi
$dir/change
586 # ROBJ10 must be handled after digests are re-computed by a deep scrub below
587 # ROBJ11 must be handled with config change before deep scrub
588 # ROBJ12 must be handled with config change before scrubs
589 # ROBJ13 must be handled before scrubs
592 echo -n bad-val
> $dir/bad-val
593 objectstore_tool
$dir 0 $objname set-attr _
$dir/bad-val ||
return 1
594 objectstore_tool
$dir 1 $objname rm-attr _ ||
return 1
599 objectstore_tool
$dir $osd $objname rm-attr _ ||
return 1
603 objectstore_tool
$dir 0 $objname rm-attr snapset ||
return 1
604 echo -n bad-val
> $dir/bad-val
605 objectstore_tool
$dir 1 $objname set-attr snapset
$dir/bad-val ||
return 1
610 local pg
=$
(get_pg
$poolname ROBJ0
)
612 inject_eio rep data
$poolname ROBJ11
$dir 0 ||
return 1 # shard 0 of [1, 0], osd.1
613 inject_eio rep mdata
$poolname ROBJ12
$dir 1 ||
return 1 # shard 1 of [1, 0], osd.0
614 inject_eio rep mdata
$poolname ROBJ13
$dir 1 ||
return 1 # shard 1 of [1, 0], osd.0
615 inject_eio rep data
$poolname ROBJ13
$dir 0 ||
return 1 # shard 0 of [1, 0], osd.1
619 rados list-inconsistent-pg
$poolname > $dir/json ||
return 1
621 test $
(jq
'. | length' $dir/json
) = "1" ||
return 1
623 test $
(jq
-r '.[0]' $dir/json
) = $pg ||
return 1
625 rados list-inconsistent-obj
$pg > $dir/json ||
return 1
626 # Get epoch for repair-get requests
627 epoch
=$
(jq .epoch
$dir/json
)
629 jq
"$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
644 "obj_size_oi_mismatch"
650 "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])",
651 "union_shard_errors": [
653 "obj_size_oi_mismatch"
682 "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])",
683 "union_shard_errors": [
711 "selected_object_info": "3:d60617f9:::ROBJ13:head(47'55 osd.0.0:54 dirty|omap|data_digest|omap_digest s 7 uv 39 dd 2ddbf8f5 od 6441854d alloc_hint [0 0 0])",
712 "union_shard_errors": [
736 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
751 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
762 "union_shard_errors": [
786 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
799 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
811 "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])",
812 "union_shard_errors": [
870 "union_shard_errors": [
891 "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])",
892 "union_shard_errors": [
916 "name": "_key1-ROBJ8"
920 "value": "val3-ROBJ8",
921 "name": "_key3-ROBJ8"
925 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
943 "value": "val1-ROBJ8",
944 "name": "_key1-ROBJ8"
948 "value": "val2-ROBJ8",
949 "name": "_key2-ROBJ8"
953 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
963 "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 66 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])",
964 "union_shard_errors": [],
966 "attr_value_mismatch",
980 "object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 67 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])",
987 "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])",
990 "obj_size_oi_mismatch"
996 "selected_object_info": "3:ffdb2004:::ROBJ9:head(102'63 client.4433.0:1 dirty|omap|data_digest|omap_digest s 1 uv 67 dd 2b63260d od 2eecc539 alloc_hint [0 0 0])",
997 "union_shard_errors": [
998 "obj_size_oi_mismatch"
1001 "object_info_inconsistency"
1016 jq
"$jqfilter" $dir/json | python
-c "$sortkeys" |
sed -e "$sedfilter" > $dir/csjson
1017 diff ${DIFFCOLOPTS} $dir/checkcsjson
$dir/csjson ||
test $getjson = "yes" ||
return 1
1018 if test $getjson = "yes"
1020 jq
'.' $dir/json
> save1.json
1023 if which jsonschema
> /dev
/null
;
1025 jsonschema
-i $dir/json
$CEPH_ROOT/doc
/rados
/command
/list-inconsistent-obj.json ||
return 1
1029 # Change data and size again because digest was recomputed
1030 echo -n ZZZ
> $dir/change
1031 rados
--pool $poolname put
$objname $dir/change
1032 # Set one to an even older value
1033 objectstore_tool
$dir 0 $objname set-attr _
$dir/robj9-oi
1034 rm $dir/oi
$dir/change
1037 objectstore_tool
$dir 1 $objname get-attr _
> $dir/oi
1038 rados
--pool $poolname setomapval
$objname key2-
$objname val2-
$objname
1039 objectstore_tool
$dir 0 $objname set-attr _
$dir/oi
1040 objectstore_tool
$dir 1 $objname set-attr _
$dir/oi
1043 inject_eio rep data
$poolname ROBJ11
$dir 0 ||
return 1 # shard 0 of [1, 0], osd.1
1044 inject_eio rep mdata
$poolname ROBJ12
$dir 1 ||
return 1 # shard 1 of [1, 0], osd.0
1045 inject_eio rep mdata
$poolname ROBJ13
$dir 1 ||
return 1 # shard 1 of [1, 0], osd.0
1046 inject_eio rep data
$poolname ROBJ13
$dir 0 ||
return 1 # shard 0 of [1, 0], osd.1
1049 rados list-inconsistent-pg
$poolname > $dir/json ||
return 1
1051 test $
(jq
'. | length' $dir/json
) = "1" ||
return 1
1053 test $
(jq
-r '.[0]' $dir/json
) = $pg ||
return 1
1055 rados list-inconsistent-obj
$pg > $dir/json ||
return 1
1056 # Get epoch for repair-get requests
1057 epoch
=$
(jq .epoch
$dir/json
)
1059 jq
"$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
1065 "data_digest": "0x2ddbf8f5",
1066 "omap_digest": "0xf5fba2c6",
1073 "data_digest": "0x2d4a11c2",
1074 "omap_digest": "0xf5fba2c6",
1077 "data_digest_mismatch_oi",
1079 "obj_size_oi_mismatch"
1085 "selected_object_info": "3:ce3f1d6a:::ROBJ1:head(47'54 osd.0.0:53 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [0 0 0])",
1086 "union_shard_errors": [
1087 "data_digest_mismatch_oi",
1089 "obj_size_oi_mismatch"
1092 "data_digest_mismatch",
1106 "data_digest": "0x2ddbf8f5",
1107 "omap_digest": "0xa8dd5adc",
1110 "omap_digest_mismatch_oi"
1116 "data_digest": "0x2ddbf8f5",
1117 "omap_digest": "0xa8dd5adc",
1120 "omap_digest_mismatch_oi"
1126 "selected_object_info": "3:b1f19cbd:::ROBJ10:head(47'51 osd.0.0:50 dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [0 0 0])",
1127 "union_shard_errors": [
1128 "omap_digest_mismatch_oi"
1142 "data_digest": "0x2ddbf8f5",
1143 "omap_digest": "0xa03cef03",
1158 "selected_object_info": "3:87abbf36:::ROBJ11:head(47'48 osd.0.0:47 dirty|omap|data_digest|omap_digest s 7 uv 33 dd 2ddbf8f5 od a03cef03 alloc_hint [0 0 0])",
1159 "union_shard_errors": [
1181 "data_digest": "0x2ddbf8f5",
1182 "omap_digest": "0x067f306a",
1189 "selected_object_info": "3:bc819597:::ROBJ12:head(47'52 osd.0.0:51 dirty|omap|data_digest|omap_digest s 7 uv 36 dd 2ddbf8f5 od 67f306a alloc_hint [0 0 0])",
1190 "union_shard_errors": [
1220 "union_shard_errors": [
1244 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1248 "data_digest": "0x2ddbf8f5",
1249 "omap_digest": "0x4f14f849",
1261 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1265 "data_digest": "0x2ddbf8f5",
1266 "omap_digest": "0x4f14f849",
1275 "union_shard_errors": [
1299 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1303 "data_digest": "0x2ddbf8f5",
1304 "omap_digest": "0x2d2a4d6e",
1314 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1318 "data_digest": "0x2ddbf8f5",
1319 "omap_digest": "0x2d2a4d6e",
1328 "selected_object_info": "3:30259878:::ROBJ15:head(47'46 osd.0.0:45 dirty|omap|data_digest|omap_digest s 7 uv 45 dd 2ddbf8f5 od 2d2a4d6e alloc_hint [0 0 0])",
1329 "union_shard_errors": [
1359 "data_digest": "0x2ddbf8f5",
1363 "omap_digest": "0x8b699207",
1381 "data_digest": "0x2ddbf8f5",
1385 "omap_digest": "0x8b699207",
1391 "union_shard_errors": [
1399 "data_digest": "0x578a4830",
1400 "omap_digest": "0xf8e11918",
1403 "data_digest_mismatch_oi"
1409 "data_digest": "0x2ddbf8f5",
1410 "omap_digest": "0xf8e11918",
1417 "selected_object_info": "3:e97ce31e:::ROBJ2:head(47'56 osd.0.0:55 dirty|omap|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od f8e11918 alloc_hint [0 0 0])",
1418 "union_shard_errors": [
1419 "data_digest_mismatch_oi"
1422 "data_digest_mismatch"
1435 "data_digest": "0x2ddbf8f5",
1436 "omap_digest": "0x00b35dfd",
1450 "selected_object_info": "3:f2a5b2a4:::ROBJ3:head(47'57 osd.0.0:56 dirty|omap|data_digest|omap_digest s 7 uv 9 dd 2ddbf8f5 od b35dfd alloc_hint [0 0 0])",
1451 "union_shard_errors": [
1466 "data_digest": "0x2ddbf8f5",
1467 "omap_digest": "0xd7178dfe",
1470 "omap_digest_mismatch_oi"
1476 "data_digest": "0x2ddbf8f5",
1477 "omap_digest": "0xe2d46ea4",
1484 "selected_object_info": "3:f4981d31:::ROBJ4:head(47'58 osd.0.0:57 dirty|omap|data_digest|omap_digest s 7 uv 12 dd 2ddbf8f5 od e2d46ea4 alloc_hint [0 0 0])",
1485 "union_shard_errors": [
1486 "omap_digest_mismatch_oi"
1489 "omap_digest_mismatch"
1502 "data_digest": "0x2ddbf8f5",
1503 "omap_digest": "0x1a862a41",
1510 "data_digest": "0x2ddbf8f5",
1511 "omap_digest": "0x06cac8f6",
1514 "omap_digest_mismatch_oi"
1520 "selected_object_info": "3:f4bfd4d1:::ROBJ5:head(47'59 osd.0.0:58 dirty|omap|data_digest|omap_digest s 7 uv 15 dd 2ddbf8f5 od 1a862a41 alloc_hint [0 0 0])",
1521 "union_shard_errors": [
1522 "omap_digest_mismatch_oi"
1525 "omap_digest_mismatch"
1538 "data_digest": "0x2ddbf8f5",
1539 "omap_digest": "0x689ee887",
1542 "omap_digest_mismatch_oi"
1548 "data_digest": "0x2ddbf8f5",
1549 "omap_digest": "0x179c919f",
1556 "selected_object_info": "3:a53c12e8:::ROBJ6:head(47'50 osd.0.0:49 dirty|omap|data_digest|omap_digest s 7 uv 18 dd 2ddbf8f5 od 179c919f alloc_hint [0 0 0])",
1557 "union_shard_errors": [
1558 "omap_digest_mismatch_oi"
1561 "omap_digest_mismatch"
1574 "data_digest": "0x2ddbf8f5",
1575 "omap_digest": "0xefced57a",
1582 "data_digest": "0x2ddbf8f5",
1583 "omap_digest": "0x6a73cc07",
1586 "omap_digest_mismatch_oi"
1592 "selected_object_info": "3:8b55fa4b:::ROBJ7:head(47'49 osd.0.0:48 dirty|omap|data_digest|omap_digest s 7 uv 21 dd 2ddbf8f5 od efced57a alloc_hint [0 0 0])",
1593 "union_shard_errors": [
1594 "omap_digest_mismatch_oi"
1597 "omap_digest_mismatch"
1619 "name": "_key1-ROBJ8"
1623 "value": "val3-ROBJ8",
1624 "name": "_key3-ROBJ8"
1628 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1632 "data_digest": "0x2ddbf8f5",
1633 "omap_digest": "0xd6be81dc",
1648 "value": "val1-ROBJ8",
1649 "name": "_key1-ROBJ8"
1653 "value": "val2-ROBJ8",
1654 "name": "_key2-ROBJ8"
1658 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1662 "data_digest": "0x2ddbf8f5",
1663 "omap_digest": "0xd6be81dc",
1670 "selected_object_info": "3:86586531:::ROBJ8:head(82'62 client.4351.0:1 dirty|omap|data_digest|omap_digest s 7 uv 66 dd 2ddbf8f5 od d6be81dc alloc_hint [0 0 0])",
1671 "union_shard_errors": [],
1673 "attr_value_mismatch",
1674 "attr_name_mismatch"
1687 "object_info": "3:ffdb2004:::ROBJ9:head(47'60 osd.0.0:59 dirty|omap|data_digest|omap_digest s 7 uv 27 dd 2ddbf8f5 od 2eecc539 alloc_hint [0 0 0])",
1688 "data_digest": "0x1f26fb26",
1689 "omap_digest": "0x2eecc539",
1692 "obj_size_oi_mismatch"
1698 "object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 68 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])",
1699 "data_digest": "0x1f26fb26",
1700 "omap_digest": "0x2eecc539",
1707 "selected_object_info": "3:ffdb2004:::ROBJ9:head(122'64 client.4532.0:1 dirty|omap|data_digest|omap_digest s 3 uv 68 dd 1f26fb26 od 2eecc539 alloc_hint [0 0 0])",
1708 "union_shard_errors": [
1709 "obj_size_oi_mismatch"
1712 "object_info_inconsistency"
1727 jq
"$jqfilter" $dir/json | python
-c "$sortkeys" |
sed -e "$sedfilter" > $dir/csjson
1728 diff ${DIFFCOLOPTS} $dir/checkcsjson
$dir/csjson ||
test $getjson = "yes" ||
return 1
1729 if test $getjson = "yes"
1731 jq
'.' $dir/json
> save2.json
1734 if which jsonschema
> /dev
/null
;
1736 jsonschema
-i $dir/json
$CEPH_ROOT/doc
/rados
/command
/list-inconsistent-obj.json ||
return 1
1739 rados rmpool
$poolname $poolname --yes-i-really-really-mean-it
1740 teardown
$dir ||
return 1
1745 # Test scrub errors for an erasure coded pool
1747 function corrupt_scrub_erasure
() {
1749 local allow_overwrites
=$2
1750 local poolname
=ecpool
1753 setup
$dir ||
return 1
1754 run_mon
$dir a ||
return 1
1755 run_mgr
$dir x ||
return 1
1756 for id
in $
(seq 0 2) ; do
1757 if [ "$allow_overwrites" = "true" ]; then
1758 run_osd_bluestore
$dir $id ||
return 1
1760 run_osd
$dir $id ||
return 1
1763 create_rbd_pool ||
return 1
1766 create_ec_pool
$poolname $allow_overwrites k
=2 m
=1 stripe_unit
=2K
--force ||
return 1
1767 wait_for_clean ||
return 1
1769 for i
in $
(seq 1 $total_objs) ; do
1771 add_something
$dir $poolname $objname ||
return 1
1773 local osd
=$
(expr $i % 2)
1777 # Size (deep scrub data_digest too)
1778 local payload
=UVWXYZZZ
1779 echo $payload > $dir/CORRUPT
1780 objectstore_tool
$dir $osd $objname set-bytes
$dir/CORRUPT ||
return 1
1785 dd if=/dev
/urandom of
=$dir/CORRUPT bs
=2048 count
=1
1786 objectstore_tool
$dir $osd $objname set-bytes
$dir/CORRUPT ||
return 1
1791 objectstore_tool
$dir $osd $objname remove ||
return 1
1795 rados
--pool $poolname setxattr
$objname key1-
$objname val1-
$objname ||
return 1
1796 rados
--pool $poolname setxattr
$objname key2-
$objname val2-
$objname ||
return 1
1799 echo -n bad-val
> $dir/bad-val
1800 objectstore_tool
$dir $osd $objname set-attr _key1-
$objname $dir/bad-val ||
return 1
1801 objectstore_tool
$dir $osd $objname rm-attr _key2-
$objname ||
return 1
1802 echo -n val3-
$objname > $dir/newval
1803 objectstore_tool
$dir $osd $objname set-attr _key3-
$objname $dir/newval ||
return 1
1804 rm $dir/bad-val
$dir/newval
1809 dd if=/dev
/urandom of
=$dir/CORRUPT bs
=2048 count
=2
1810 objectstore_tool
$dir $osd $objname set-bytes
$dir/CORRUPT ||
return 1
1816 local pg
=$
(get_pg
$poolname EOBJ0
)
1820 rados list-inconsistent-pg
$poolname > $dir/json ||
return 1
1822 test $
(jq
'. | length' $dir/json
) = "1" ||
return 1
1824 test $
(jq
-r '.[0]' $dir/json
) = $pg ||
return 1
1826 rados list-inconsistent-obj
$pg > $dir/json ||
return 1
1827 # Get epoch for repair-get requests
1828 epoch
=$
(jq .epoch
$dir/json
)
1830 jq
"$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
1847 "obj_size_oi_mismatch"
1860 "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1861 "union_shard_errors": [
1863 "obj_size_oi_mismatch"
1901 "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
1902 "union_shard_errors": [
1926 "name": "_key1-EOBJ4"
1930 "value": "val3-EOBJ4",
1931 "name": "_key3-EOBJ4"
1935 "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
1940 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1964 "value": "val1-EOBJ4",
1965 "name": "_key1-EOBJ4"
1969 "value": "val2-EOBJ4",
1970 "name": "_key2-EOBJ4"
1974 "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
1979 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
1998 "value": "val1-EOBJ4",
1999 "name": "_key1-EOBJ4"
2003 "value": "val2-EOBJ4",
2004 "name": "_key2-EOBJ4"
2008 "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2013 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2019 "selected_object_info": "3:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2020 "union_shard_errors": [],
2022 "attr_value_mismatch",
2023 "attr_name_mismatch"
2047 "obj_size_oi_mismatch"
2060 "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2061 "union_shard_errors": [
2063 "obj_size_oi_mismatch"
2081 jq
"$jqfilter" $dir/json | python
-c "$sortkeys" |
sed -e "$sedfilter" > $dir/csjson
2082 diff ${DIFFCOLOPTS} $dir/checkcsjson
$dir/csjson ||
test $getjson = "yes" ||
return 1
2083 if test $getjson = "yes"
2085 jq
'.' $dir/json
> save3.json
2088 if which jsonschema
> /dev
/null
;
2090 jsonschema
-i $dir/json
$CEPH_ROOT/doc
/rados
/command
/list-inconsistent-obj.json ||
return 1
2095 rados list-inconsistent-pg
$poolname > $dir/json ||
return 1
2097 test $
(jq
'. | length' $dir/json
) = "1" ||
return 1
2099 test $
(jq
-r '.[0]' $dir/json
) = $pg ||
return 1
2101 rados list-inconsistent-obj
$pg > $dir/json ||
return 1
2102 # Get epoch for repair-get requests
2103 epoch
=$
(jq .epoch
$dir/json
)
2105 if [ "$allow_overwrites" = "true" ]
2107 jq
"$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
2113 "data_digest": "0x00000000",
2114 "omap_digest": "0xffffffff",
2127 "obj_size_oi_mismatch"
2133 "data_digest": "0x00000000",
2134 "omap_digest": "0xffffffff",
2142 "selected_object_info": "3:9175b684:::EOBJ1:head(27'1 client.4155.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2143 "union_shard_errors": [
2146 "obj_size_oi_mismatch"
2162 "data_digest": "0x00000000",
2163 "omap_digest": "0xffffffff",
2179 "data_digest": "0x00000000",
2180 "omap_digest": "0xffffffff",
2188 "selected_object_info": "3:b197b25d:::EOBJ3:head(41'3 client.4199.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2189 "union_shard_errors": [
2213 "name": "_key1-EOBJ4"
2217 "value": "val3-EOBJ4",
2218 "name": "_key3-EOBJ4"
2222 "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2227 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2231 "data_digest": "0x00000000",
2232 "omap_digest": "0xffffffff",
2248 "value": "val1-EOBJ4",
2249 "name": "_key1-EOBJ4"
2253 "value": "val2-EOBJ4",
2254 "name": "_key2-EOBJ4"
2258 "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2263 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2267 "data_digest": "0x00000000",
2268 "omap_digest": "0xffffffff",
2284 "value": "val1-EOBJ4",
2285 "name": "_key1-EOBJ4"
2289 "value": "val2-EOBJ4",
2290 "name": "_key2-EOBJ4"
2294 "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2299 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2303 "data_digest": "0x00000000",
2304 "omap_digest": "0xffffffff",
2312 "selected_object_info": "3:5e723e06:::EOBJ4:head(48'6 client.4223.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2313 "union_shard_errors": [],
2315 "attr_value_mismatch",
2316 "attr_name_mismatch"
2329 "data_digest": "0x00000000",
2330 "omap_digest": "0xffffffff",
2338 "data_digest": "0x00000000",
2339 "omap_digest": "0xffffffff",
2343 "obj_size_oi_mismatch"
2350 "data_digest": "0x00000000",
2351 "omap_digest": "0xffffffff",
2359 "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4288.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2360 "union_shard_errors": [
2362 "obj_size_oi_mismatch"
2382 jq
"$jqfilter" << EOF | python -c "$sortkeys" | sed -e "$sedfilter" > $dir/checkcsjson
2388 "data_digest": "0x04cfa72f",
2389 "omap_digest": "0xffffffff",
2402 "obj_size_oi_mismatch"
2408 "data_digest": "0x04cfa72f",
2409 "omap_digest": "0xffffffff",
2417 "selected_object_info": "3:9175b684:::EOBJ1:head(21'1 client.4179.0:1 dirty|data_digest|omap_digest s 7 uv 1 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2418 "union_shard_errors": [
2421 "obj_size_oi_mismatch"
2446 "data_digest": "0x04cfa72f",
2447 "omap_digest": "0xffffffff",
2455 "data_digest": "0x04cfa72f",
2456 "omap_digest": "0xffffffff",
2464 "selected_object_info": "3:9babd184:::EOBJ2:head(29'2 client.4217.0:1 dirty|data_digest|omap_digest s 7 uv 2 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2465 "union_shard_errors": [
2480 "data_digest": "0x04cfa72f",
2481 "omap_digest": "0xffffffff",
2497 "data_digest": "0x04cfa72f",
2498 "omap_digest": "0xffffffff",
2506 "selected_object_info": "3:b197b25d:::EOBJ3:head(37'3 client.4251.0:1 dirty|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2507 "union_shard_errors": [
2531 "name": "_key1-EOBJ4"
2535 "value": "val3-EOBJ4",
2536 "name": "_key3-EOBJ4"
2540 "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2545 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2549 "data_digest": "0x04cfa72f",
2550 "omap_digest": "0xffffffff",
2563 "omap_digest": "0xffffffff",
2564 "data_digest": "0x04cfa72f",
2573 "value": "val1-EOBJ4",
2574 "name": "_key1-EOBJ4"
2578 "value": "val2-EOBJ4",
2579 "name": "_key2-EOBJ4"
2583 "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2588 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2599 "omap_digest": "0xffffffff",
2600 "data_digest": "0x04cfa72f",
2609 "value": "val1-EOBJ4",
2610 "name": "_key1-EOBJ4"
2614 "value": "val2-EOBJ4",
2615 "name": "_key2-EOBJ4"
2619 "value": "AQEYAAAAAAgAAAAAAAADAAAAL6fPBLB8dlsvp88E",
2624 "value": "AwIdAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
2630 "selected_object_info": "3:5e723e06:::EOBJ4:head(45'6 client.4289.0:1 dirty|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2631 "union_shard_errors": [],
2633 "attr_value_mismatch",
2634 "attr_name_mismatch"
2647 "data_digest": "0x04cfa72f",
2648 "omap_digest": "0xffffffff",
2661 "obj_size_oi_mismatch"
2667 "data_digest": "0x04cfa72f",
2668 "omap_digest": "0xffffffff",
2676 "selected_object_info": "3:8549dfb5:::EOBJ5:head(65'7 client.4441.0:1 dirty|data_digest|omap_digest s 7 uv 7 dd 2ddbf8f5 od ffffffff alloc_hint [0 0 0])",
2677 "union_shard_errors": [
2680 "obj_size_oi_mismatch"
2700 jq
"$jqfilter" $dir/json | python
-c "$sortkeys" |
sed -e "$sedfilter" > $dir/csjson
2701 diff ${DIFFCOLOPTS} $dir/checkcsjson
$dir/csjson ||
test $getjson = "yes" ||
return 1
2702 if test $getjson = "yes"
2704 if [ "$allow_overwrites" = "true" ]
2710 jq
'.' $dir/json
> save
${num}.json
2713 if which jsonschema
> /dev
/null
;
2715 jsonschema
-i $dir/json
$CEPH_ROOT/doc
/rados
/command
/list-inconsistent-obj.json ||
return 1
2718 rados rmpool
$poolname $poolname --yes-i-really-really-mean-it
2719 teardown
$dir ||
return 1
2722 function TEST_corrupt_scrub_erasure_appends
() {
2723 corrupt_scrub_erasure
$1 false
2726 function TEST_corrupt_scrub_erasure_overwrites
() {
2727 if [ "$use_ec_overwrite" = "true" ]; then
2728 corrupt_scrub_erasure
$1 true
2733 # Test to make sure that a periodic scrub won't cause deep-scrub info to be lost
2735 function TEST_periodic_scrub_replicated
() {
2737 local poolname
=psr_pool
2740 setup
$dir ||
return 1
2741 run_mon
$dir a
--osd_pool_default_size=2 ||
return 1
2742 run_mgr
$dir x ||
return 1
2743 local ceph_osd_args
="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 "
2744 ceph_osd_args
+="--osd_scrub_backoff_ratio=0"
2745 run_osd
$dir 0 $ceph_osd_args ||
return 1
2746 run_osd
$dir 1 $ceph_osd_args ||
return 1
2747 create_rbd_pool ||
return 1
2748 wait_for_clean ||
return 1
2750 create_pool
$poolname 1 1 ||
return 1
2751 wait_for_clean ||
return 1
2754 add_something
$dir $poolname $objname scrub ||
return 1
2755 local primary
=$
(get_primary
$poolname $objname)
2756 local pg
=$
(get_pg
$poolname $objname)
2758 # Add deep-scrub only error
2759 local payload
=UVWXYZ
2760 echo $payload > $dir/CORRUPT
2761 # Uses $ceph_osd_args for osd restart
2762 objectstore_tool
$dir $osd $objname set-bytes
$dir/CORRUPT ||
return 1
2764 # No scrub information available, so expect failure
2766 ! rados list-inconsistent-obj
$pg | jq
'.' ||
return 1
2769 pg_deep_scrub
$pg ||
return 1
2771 # Make sure bad object found
2772 rados list-inconsistent-obj
$pg | jq
'.' |
grep -q $objname ||
return 1
2775 local last_scrub
=$
(get_last_scrub_stamp
$pg)
2776 # Fake a schedule scrub
2777 CEPH_ARGS
='' ceph
--admin-daemon $
(get_asok_path osd.
${primary}) \
2778 trigger_scrub
$pg ||
return 1
2779 # Wait for schedule regular scrub
2780 wait_for_scrub
$pg "$last_scrub"
2782 # It needed to be upgraded
2783 grep -q "Deep scrub errors, upgrading scrub to deep-scrub" $dir/osd.
${primary}.log ||
return 1
2785 # Bad object still known
2786 rados list-inconsistent-obj
$pg | jq
'.' |
grep -q $objname ||
return 1
2788 # Can't upgrade with this set
2789 ceph osd
set nodeep-scrub
2790 # Let map change propagate to OSDs
2794 # Fake a schedule scrub
2795 CEPH_ARGS
='' ceph
--admin-daemon $
(get_asok_path osd.
${primary}) \
2796 trigger_scrub
$pg ||
return 1
2797 # Wait for schedule regular scrub
2798 # to notice scrub and skip it
2800 for i
in $
(seq 14 -1 0)
2803 ! grep -q "Regular scrub skipped due to deep-scrub errors and nodeep-scrub set" $dir/osd.
${primary}.log ||
{ found
=true
; break; }
2804 echo Time left
: $i seconds
2806 test $found = "true" ||
return 1
2808 # Bad object still known
2809 rados list-inconsistent-obj
$pg | jq
'.' |
grep -q $objname ||
return 1
2812 # Request a regular scrub and it will be done
2814 grep -q "Regular scrub request, deep-scrub details will be lost" $dir/osd.
${primary}.log ||
return 1
2816 # deep-scrub error is no longer present
2817 rados list-inconsistent-obj
$pg | jq
'.' |
grep -qv $objname ||
return 1
2821 main osd-scrub-repair
"$@"
2824 # compile-command: "cd ../.. ; make -j4 && \
2825 # test/osd/osd-scrub-repair.sh # TEST_corrupt_and_repair_replicated"