]> git.proxmox.com Git - ceph.git/blob - ceph/qa/standalone/scrub/osd-scrub-repair.sh
projects / ceph.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
import quincy beta 17.1.0
[ceph.git] / ceph / qa / standalone / scrub / osd-scrub-repair.sh
1 #!/usr/bin/env bash
2 #
3 # Copyright (C) 2014 Red Hat <contact@redhat.com>
4 #
5 # Author: Loic Dachary <loic@dachary.org>
6 #
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU Library Public License as published by
9 # the Free Software Foundation; either version 2, or (at your option)
10 # any later version.
11 #
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Library Public License for more details.
16 #
17 set -x
18 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
19
20 if [ `uname` = FreeBSD ]; then
21 # erasure coding overwrites are only tested on Bluestore
22 # erasure coding on filestore is unsafe
23 # http://docs.ceph.com/en/latest/rados/operations/erasure-code/#erasure-coding-with-overwrites
24 use_ec_overwrite=false
25 else
26 use_ec_overwrite=true
27 fi
28
29 # Test development and debugging
30 # Set to "yes" in order to ignore diff errors and save results to update test
31 getjson="no"
32
33 # Filter out mtime and local_mtime dates, version, prior_version and last_reqid (client) from any object_info.
34 jqfilter='def walk(f):
35 . as $in
36 | if type == "object" then
37 reduce keys[] as $key
38 ( {}; . + { ($key): ($in[$key] | walk(f)) } ) | f
39 elif type == "array" then map( walk(f) ) | f
40 else f
41 end;
42 walk(if type == "object" then del(.mtime) else . end)
43 | walk(if type == "object" then del(.local_mtime) else . end)
44 | walk(if type == "object" then del(.last_reqid) else . end)
45 | walk(if type == "object" then del(.version) else . end)
46 | walk(if type == "object" then del(.prior_version) else . end)'
47
48 sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print(json.dumps(ud, sort_keys=True, indent=2))'
49
50 function run() {
51 local dir=$1
52 shift
53
54 export CEPH_MON="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one
55 export CEPH_ARGS
56 CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
57 CEPH_ARGS+="--mon-host=$CEPH_MON "
58 CEPH_ARGS+="--osd-skip-data-digest=false "
59
60 export -n CEPH_CLI_TEST_DUP_COMMAND
61 local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
62 for func in $funcs ; do
63 setup $dir || return 1
64 $func $dir || return 1
65 teardown $dir || return 1
66 done
67 }
68
69 function add_something() {
70 local dir=$1
71 local poolname=$2
72 local obj=${3:-SOMETHING}
73 local scrub=${4:-noscrub}
74
75 if [ "$scrub" = "noscrub" ];
76 then
77 ceph osd set noscrub || return 1
78 ceph osd set nodeep-scrub || return 1
79 else
80 ceph osd unset noscrub || return 1
81 ceph osd unset nodeep-scrub || return 1
82 fi
83
84 local payload=ABCDEF
85 echo $payload > $dir/ORIGINAL
86 rados --pool $poolname put $obj $dir/ORIGINAL || return 1
87 }
88
89 #
90 # Corrupt one copy of a replicated pool
91 #
92 function TEST_corrupt_and_repair_replicated() {
93 local dir=$1
94 local poolname=rbd
95
96 run_mon $dir a --osd_pool_default_size=2 || return 1
97 run_mgr $dir x || return 1
98 run_osd $dir 0 || return 1
99 run_osd $dir 1 || return 1
100 create_rbd_pool || return 1
101 wait_for_clean || return 1
102
103 add_something $dir $poolname || return 1
104 corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1
105 # Reproduces http://tracker.ceph.com/issues/8914
106 corrupt_and_repair_one $dir $poolname $(get_primary $poolname SOMETHING) || return 1
107 }
108
109 #
110 # Allow repair to be scheduled when some recovering is still undergoing on the same OSD
111 #
112 function TEST_allow_repair_during_recovery() {
113 local dir=$1
114 local poolname=rbd
115
116 run_mon $dir a --osd_pool_default_size=2 || return 1
117 run_mgr $dir x || return 1
118 run_osd $dir 0 --osd_scrub_during_recovery=false \
119 --osd_repair_during_recovery=true \
120 --osd_debug_pretend_recovery_active=true || return 1
121 run_osd $dir 1 --osd_scrub_during_recovery=false \
122 --osd_repair_during_recovery=true \
123 --osd_debug_pretend_recovery_active=true || return 1
124 create_rbd_pool || return 1
125 wait_for_clean || return 1
126
127 add_something $dir $poolname || return 1
128 corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1
129 }
130
131 #
132 # Skip non-repair scrub correctly during recovery
133 #
134 function TEST_skip_non_repair_during_recovery() {
135 local dir=$1
136 local poolname=rbd
137
138 run_mon $dir a --osd_pool_default_size=2 || return 1
139 run_mgr $dir x || return 1
140 run_osd $dir 0 --osd_scrub_during_recovery=false \
141 --osd_repair_during_recovery=true \
142 --osd_debug_pretend_recovery_active=true || return 1
143 run_osd $dir 1 --osd_scrub_during_recovery=false \
144 --osd_repair_during_recovery=true \
145 --osd_debug_pretend_recovery_active=true || return 1
146 create_rbd_pool || return 1
147 wait_for_clean || return 1
148
149 add_something $dir $poolname || return 1
150 scrub_and_not_schedule $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1
151 }
152
153 function scrub_and_not_schedule() {
154 local dir=$1
155 local poolname=$2
156 local osd=$3
157
158 #
159 # 1) start a non-repair scrub
160 #
161 local pg=$(get_pg $poolname SOMETHING)
162 local last_scrub=$(get_last_scrub_stamp $pg)
163 ceph pg scrub $pg
164
165 #
166 # 2) Assure the scrub is not scheduled
167 #
168 for ((i=0; i < 3; i++)); do
169 if test "$(get_last_scrub_stamp $pg)" '>' "$last_scrub" ; then
170 return 1
171 fi
172 sleep 1
173 done
174
175 #
176 # 3) Access to the file must OK
177 #
178 objectstore_tool $dir $osd SOMETHING list-attrs || return 1
179 rados --pool $poolname get SOMETHING $dir/COPY || return 1
180 diff $dir/ORIGINAL $dir/COPY || return 1
181 }
182
183 function corrupt_and_repair_two() {
184 local dir=$1
185 local poolname=$2
186 local first=$3
187 local second=$4
188
189 #
190 # 1) remove the corresponding file from the OSDs
191 #
192 pids=""
193 run_in_background pids objectstore_tool $dir $first SOMETHING remove
194 run_in_background pids objectstore_tool $dir $second SOMETHING remove
195 wait_background pids
196 return_code=$?
197 if [ $return_code -ne 0 ]; then return $return_code; fi
198
199 #
200 # 2) repair the PG
201 #
202 local pg=$(get_pg $poolname SOMETHING)
203 repair $pg
204 #
205 # 3) The files must be back
206 #
207 pids=""
208 run_in_background pids objectstore_tool $dir $first SOMETHING list-attrs
209 run_in_background pids objectstore_tool $dir $second SOMETHING list-attrs
210 wait_background pids
211 return_code=$?
212 if [ $return_code -ne 0 ]; then return $return_code; fi
213
214 rados --pool $poolname get SOMETHING $dir/COPY || return 1
215 diff $dir/ORIGINAL $dir/COPY || return 1
216 }
217
218 #
219 # 1) add an object
220 # 2) remove the corresponding file from a designated OSD
221 # 3) repair the PG
222 # 4) check that the file has been restored in the designated OSD
223 #
224 function corrupt_and_repair_one() {
225 local dir=$1
226 local poolname=$2
227 local osd=$3
228
229 #
230 # 1) remove the corresponding file from the OSD
231 #
232 objectstore_tool $dir $osd SOMETHING remove || return 1
233 #
234 # 2) repair the PG
235 #
236 local pg=$(get_pg $poolname SOMETHING)
237 repair $pg
238 #
239 # 3) The file must be back
240 #
241 objectstore_tool $dir $osd SOMETHING list-attrs || return 1
242 rados --pool $poolname get SOMETHING $dir/COPY || return 1
243 diff $dir/ORIGINAL $dir/COPY || return 1
244 }
245
246 function corrupt_and_repair_erasure_coded() {
247 local dir=$1
248 local poolname=$2
249
250 add_something $dir $poolname || return 1
251
252 local primary=$(get_primary $poolname SOMETHING)
253 local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
254 local not_primary_first=${osds[0]}
255 local not_primary_second=${osds[1]}
256
257 # Reproduces http://tracker.ceph.com/issues/10017
258 corrupt_and_repair_one $dir $poolname $primary || return 1
259 # Reproduces http://tracker.ceph.com/issues/10409
260 corrupt_and_repair_one $dir $poolname $not_primary_first || return 1
261 corrupt_and_repair_two $dir $poolname $not_primary_first $not_primary_second || return 1
262 corrupt_and_repair_two $dir $poolname $primary $not_primary_first || return 1
263
264 }
265
266 function auto_repair_erasure_coded() {
267 local dir=$1
268 local allow_overwrites=$2
269 local poolname=ecpool
270
271 # Launch a cluster with 5 seconds scrub interval
272 run_mon $dir a || return 1
273 run_mgr $dir x || return 1
274 local ceph_osd_args="--osd-scrub-auto-repair=true \
275 --osd-deep-scrub-interval=5 \
276 --osd-scrub-max-interval=5 \
277 --osd-scrub-min-interval=5 \
278 --osd-scrub-interval-randomize-ratio=0"
279 for id in $(seq 0 2) ; do
280 if [ "$allow_overwrites" = "true" ]; then
281 run_osd $dir $id $ceph_osd_args || return 1
282 else
283 run_osd_filestore $dir $id $ceph_osd_args || return 1
284 fi
285 done
286 create_rbd_pool || return 1
287 wait_for_clean || return 1
288
289 # Create an EC pool
290 create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
291
292 # Put an object
293 local payload=ABCDEF
294 echo $payload > $dir/ORIGINAL
295 rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
296
297 # Remove the object from one shard physically
298 # Restarted osd get $ceph_osd_args passed
299 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
300 # Wait for auto repair
301 local pgid=$(get_pg $poolname SOMETHING)
302 wait_for_scrub $pgid "$(get_last_scrub_stamp $pgid)"
303 wait_for_clean || return 1
304 # Verify - the file should be back
305 # Restarted osd get $ceph_osd_args passed
306 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
307 rados --pool $poolname get SOMETHING $dir/COPY || return 1
308 diff $dir/ORIGINAL $dir/COPY || return 1
309 }
310
311 function TEST_auto_repair_erasure_coded_appends() {
312 auto_repair_erasure_coded $1 false
313 }
314
315 function TEST_auto_repair_erasure_coded_overwrites() {
316 if [ "$use_ec_overwrite" = "true" ]; then
317 auto_repair_erasure_coded $1 true
318 fi
319 }
320
321 # initiate a scrub, then check for the (expected) 'scrubbing' and the
322 # (not expected until an error was identified) 'repair'
323 # Arguments: osd#, pg, sleep time
324 function initiate_and_fetch_state() {
325 local the_osd="osd.$1"
326 local pgid=$2
327 local last_scrub=$(get_last_scrub_stamp $pgid)
328
329 set_config "osd" "$1" "osd_scrub_sleep" "$3"
330 set_config "osd" "$1" "osd_scrub_auto_repair" "true"
331
332 flush_pg_stats
333 date --rfc-3339=ns
334
335 # note: must initiate a "regular" (periodic) deep scrub - not an operator-initiated one
336 env CEPH_ARGS= ceph --format json daemon $(get_asok_path $the_osd) deep_scrub "$pgid"
337 env CEPH_ARGS= ceph --format json daemon $(get_asok_path $the_osd) scrub "$pgid"
338
339 # wait for 'scrubbing' to appear
340 for ((i=0; i < 80; i++)); do
341
342 st=`ceph pg $pgid query --format json | jq '.state' `
343 echo $i ") state now: " $st
344
345 case "$st" in
346 *scrubbing*repair* ) echo "found scrub+repair"; return 1;; # PR #41258 should have prevented this
347 *scrubbing* ) echo "found scrub"; return 0;;
348 *inconsistent* ) echo "Got here too late. Scrub has already finished"; return 1;;
349 *recovery* ) echo "Got here too late. Scrub has already finished."; return 1;;
350 * ) echo $st;;
351 esac
352
353 if [ $((i % 10)) == 4 ]; then
354 echo "loop --------> " $i
355 fi
356 sleep 0.3
357 done
358
359 echo "Timeout waiting for deep-scrub of " $pgid " on " $the_osd " to start"
360 return 1
361 }
362
363 function wait_end_of_scrub() { # osd# pg
364 local the_osd="osd.$1"
365 local pgid=$2
366
367 for ((i=0; i < 40; i++)); do
368 st=`ceph pg $pgid query --format json | jq '.state' `
369 echo "wait-scrub-end state now: " $st
370 [[ $st =~ (.*scrubbing.*) ]] || break
371 if [ $((i % 5)) == 4 ] ; then
372 flush_pg_stats
373 fi
374 sleep 0.3
375 done
376
377 if [[ $st =~ (.*scrubbing.*) ]]
378 then
379 # a timeout
380 return 1
381 fi
382 return 0
383 }
384
385
386 function TEST_auto_repair_bluestore_tag() {
387 local dir=$1
388 local poolname=testpool
389
390 # Launch a cluster with 3 seconds scrub interval
391 run_mon $dir a || return 1
392 run_mgr $dir x || return 1
393 # Set scheduler to "wpq" until there's a reliable way to query scrub states
394 # with "--osd-scrub-sleep" set to 0. The "mclock_scheduler" overrides the
395 # scrub sleep to 0 and as a result the checks in the test fail.
396 local ceph_osd_args="--osd-scrub-auto-repair=true \
397 --osd_deep_scrub_randomize_ratio=0 \
398 --osd-scrub-interval-randomize-ratio=0 \
399 --osd-op-queue=wpq"
400 for id in $(seq 0 2) ; do
401 run_osd $dir $id $ceph_osd_args || return 1
402 done
403
404 create_pool $poolname 1 1 || return 1
405 ceph osd pool set $poolname size 2
406 wait_for_clean || return 1
407
408 # Put an object
409 local payload=ABCDEF
410 echo $payload > $dir/ORIGINAL
411 rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
412
413 # Remove the object from one shard physically
414 # Restarted osd get $ceph_osd_args passed
415 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
416
417 local pgid=$(get_pg $poolname SOMETHING)
418 local primary=$(get_primary $poolname SOMETHING)
419 echo "Affected PG " $pgid " w/ primary " $primary
420 local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
421 initiate_and_fetch_state $primary $pgid "3.0"
422 r=$?
423 echo "initiate_and_fetch_state ret: " $r
424 set_config "osd" "$1" "osd_scrub_sleep" "0"
425 if [ $r -ne 0 ]; then
426 return 1
427 fi
428
429 wait_end_of_scrub "$primary" "$pgid" || return 1
430 ceph pg dump pgs
431
432 # Verify - the file should be back
433 # Restarted osd get $ceph_osd_args passed
434 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
435 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING get-bytes $dir/COPY || return 1
436 diff $dir/ORIGINAL $dir/COPY || return 1
437 grep scrub_finish $dir/osd.${primary}.log
438 }
439
440
441 function TEST_auto_repair_bluestore_basic() {
442 local dir=$1
443 local poolname=testpool
444
445 # Launch a cluster with 5 seconds scrub interval
446 run_mon $dir a || return 1
447 run_mgr $dir x || return 1
448 local ceph_osd_args="--osd-scrub-auto-repair=true \
449 --osd_deep_scrub_randomize_ratio=0 \
450 --osd-scrub-interval-randomize-ratio=0"
451 for id in $(seq 0 2) ; do
452 run_osd $dir $id $ceph_osd_args || return 1
453 done
454
455 create_pool $poolname 1 1 || return 1
456 ceph osd pool set $poolname size 2
457 wait_for_clean || return 1
458
459 # Put an object
460 local payload=ABCDEF
461 echo $payload > $dir/ORIGINAL
462 rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
463
464 # Remove the object from one shard physically
465 # Restarted osd get $ceph_osd_args passed
466 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
467
468 local pgid=$(get_pg $poolname SOMETHING)
469 local primary=$(get_primary $poolname SOMETHING)
470 local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
471 ceph tell $pgid deep_scrub
472 ceph tell $pgid scrub
473
474 # Wait for auto repair
475 wait_for_scrub $pgid "$last_scrub_stamp" || return 1
476 wait_for_clean || return 1
477 ceph pg dump pgs
478 # Verify - the file should be back
479 # Restarted osd get $ceph_osd_args passed
480 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
481 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING get-bytes $dir/COPY || return 1
482 diff $dir/ORIGINAL $dir/COPY || return 1
483 grep scrub_finish $dir/osd.${primary}.log
484 }
485
486 function TEST_auto_repair_bluestore_scrub() {
487 local dir=$1
488 local poolname=testpool
489
490 # Launch a cluster with 5 seconds scrub interval
491 run_mon $dir a || return 1
492 run_mgr $dir x || return 1
493 local ceph_osd_args="--osd-scrub-auto-repair=true \
494 --osd_deep_scrub_randomize_ratio=0 \
495 --osd-scrub-interval-randomize-ratio=0 \
496 --osd-scrub-backoff-ratio=0"
497 for id in $(seq 0 2) ; do
498 run_osd $dir $id $ceph_osd_args || return 1
499 done
500
501 create_pool $poolname 1 1 || return 1
502 ceph osd pool set $poolname size 2
503 wait_for_clean || return 1
504
505 # Put an object
506 local payload=ABCDEF
507 echo $payload > $dir/ORIGINAL
508 rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1
509
510 # Remove the object from one shard physically
511 # Restarted osd get $ceph_osd_args passed
512 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1
513
514 local pgid=$(get_pg $poolname SOMETHING)
515 local primary=$(get_primary $poolname SOMETHING)
516 local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
517 ceph tell $pgid scrub
518
519 # Wait for scrub -> auto repair
520 wait_for_scrub $pgid "$last_scrub_stamp" || return 1
521 ceph pg dump pgs
522 # Actually this causes 2 scrubs, so we better wait a little longer
523 sleep 5
524 wait_for_clean || return 1
525 ceph pg dump pgs
526 # Verify - the file should be back
527 # Restarted osd get $ceph_osd_args passed
528 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1
529 rados --pool $poolname get SOMETHING $dir/COPY || return 1
530 diff $dir/ORIGINAL $dir/COPY || return 1
531 grep scrub_finish $dir/osd.${primary}.log
532
533 # This should have caused 1 object to be repaired
534 COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired')
535 test "$COUNT" = "1" || return 1
536 }
537
538 function TEST_auto_repair_bluestore_failed() {
539 local dir=$1
540 local poolname=testpool
541
542 # Launch a cluster with 5 seconds scrub interval
543 run_mon $dir a || return 1
544 run_mgr $dir x || return 1
545 local ceph_osd_args="--osd-scrub-auto-repair=true \
546 --osd_deep_scrub_randomize_ratio=0 \
547 --osd-scrub-interval-randomize-ratio=0"
548 for id in $(seq 0 2) ; do
549 run_osd $dir $id $ceph_osd_args || return 1
550 done
551
552 create_pool $poolname 1 1 || return 1
553 ceph osd pool set $poolname size 2
554 wait_for_clean || return 1
555
556 # Put an object
557 local payload=ABCDEF
558 echo $payload > $dir/ORIGINAL
559 for i in $(seq 1 10)
560 do
561 rados --pool $poolname put obj$i $dir/ORIGINAL || return 1
562 done
563
564 # Remove the object from one shard physically
565 # Restarted osd get $ceph_osd_args passed
566 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) obj1 remove || return 1
567 # obj2 can't be repaired
568 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) obj2 remove || return 1
569 objectstore_tool $dir $(get_primary $poolname SOMETHING) obj2 rm-attr _ || return 1
570
571 local pgid=$(get_pg $poolname obj1)
572 local primary=$(get_primary $poolname obj1)
573 local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
574 ceph tell $pgid deep_scrub
575 ceph tell $pgid scrub
576
577 # Wait for auto repair
578 wait_for_scrub $pgid "$last_scrub_stamp" || return 1
579 wait_for_clean || return 1
580 flush_pg_stats
581 grep scrub_finish $dir/osd.${primary}.log
582 grep -q "scrub_finish.*still present after re-scrub" $dir/osd.${primary}.log || return 1
583 ceph pg dump pgs
584 ceph pg dump pgs | grep -q "^${pgid}.*+failed_repair" || return 1
585
586 # Verify - obj1 should be back
587 # Restarted osd get $ceph_osd_args passed
588 objectstore_tool $dir $(get_not_primary $poolname obj1) obj1 list-attrs || return 1
589 rados --pool $poolname get obj1 $dir/COPY || return 1
590 diff $dir/ORIGINAL $dir/COPY || return 1
591 grep scrub_finish $dir/osd.${primary}.log
592
593 # Make it repairable
594 objectstore_tool $dir $(get_primary $poolname SOMETHING) obj2 remove || return 1
595 repair $pgid
596 sleep 2
597
598 flush_pg_stats
599 ceph pg dump pgs
600 ceph pg dump pgs | grep -q -e "^${pgid}.* active+clean " -e "^${pgid}.* active+clean+wait " || return 1
601 grep scrub_finish $dir/osd.${primary}.log
602 }
603
604 function TEST_auto_repair_bluestore_failed_norecov() {
605 local dir=$1
606 local poolname=testpool
607
608 # Launch a cluster with 5 seconds scrub interval
609 run_mon $dir a || return 1
610 run_mgr $dir x || return 1
611 local ceph_osd_args="--osd-scrub-auto-repair=true \
612 --osd_deep_scrub_randomize_ratio=0 \
613 --osd-scrub-interval-randomize-ratio=0"
614 for id in $(seq 0 2) ; do
615 run_osd $dir $id $ceph_osd_args || return 1
616 done
617
618 create_pool $poolname 1 1 || return 1
619 ceph osd pool set $poolname size 2
620 wait_for_clean || return 1
621
622 # Put an object
623 local payload=ABCDEF
624 echo $payload > $dir/ORIGINAL
625 for i in $(seq 1 10)
626 do
627 rados --pool $poolname put obj$i $dir/ORIGINAL || return 1
628 done
629
630 # Remove the object from one shard physically
631 # Restarted osd get $ceph_osd_args passed
632 # obj1 can't be repaired
633 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) obj1 remove || return 1
634 objectstore_tool $dir $(get_primary $poolname SOMETHING) obj1 rm-attr _ || return 1
635 # obj2 can't be repaired
636 objectstore_tool $dir $(get_not_primary $poolname SOMETHING) obj2 remove || return 1
637 objectstore_tool $dir $(get_primary $poolname SOMETHING) obj2 rm-attr _ || return 1
638
639 local pgid=$(get_pg $poolname obj1)
640 local primary=$(get_primary $poolname obj1)
641 local last_scrub_stamp="$(get_last_scrub_stamp $pgid)"
642 ceph tell $pgid deep_scrub
643 ceph tell $pgid scrub
644
645 # Wait for auto repair
646 wait_for_scrub $pgid "$last_scrub_stamp" || return 1
647 wait_for_clean || return 1
648 flush_pg_stats
649 grep -q "scrub_finish.*present with no repair possible" $dir/osd.${primary}.log || return 1
650 ceph pg dump pgs
651 ceph pg dump pgs | grep -q "^${pgid}.*+failed_repair" || return 1
652 }
653
654 function TEST_repair_stats() {
655 local dir=$1
656 local poolname=testpool
657 local OSDS=2
658 local OBJS=30
659 # This need to be an even number
660 local REPAIRS=20
661
662 # Launch a cluster with 5 seconds scrub interval
663 run_mon $dir a || return 1
664 run_mgr $dir x || return 1
665 local ceph_osd_args="--osd_deep_scrub_randomize_ratio=0 \
666 --osd-scrub-interval-randomize-ratio=0"
667 for id in $(seq 0 $(expr $OSDS - 1)) ; do
668 run_osd $dir $id $ceph_osd_args || return 1
669 done
670
671 create_pool $poolname 1 1 || return 1
672 ceph osd pool set $poolname size 2
673 wait_for_clean || return 1
674
675 # Put an object
676 local payload=ABCDEF
677 echo $payload > $dir/ORIGINAL
678 for i in $(seq 1 $OBJS)
679 do
680 rados --pool $poolname put obj$i $dir/ORIGINAL || return 1
681 done
682
683 # Remove the object from one shard physically
684 # Restarted osd get $ceph_osd_args passed
685 local other=$(get_not_primary $poolname obj1)
686 local pgid=$(get_pg $poolname obj1)
687 local primary=$(get_primary $poolname obj1)
688
689 kill_daemons $dir TERM osd.$other >&2 < /dev/null || return 1
690 kill_daemons $dir TERM osd.$primary >&2 < /dev/null || return 1
691 for i in $(seq 1 $REPAIRS)
692 do
693 # Remove from both osd.0 and osd.1
694 OSD=$(expr $i % 2)
695 _objectstore_tool_nodown $dir $OSD obj$i remove || return 1
696 done
697 activate_osd $dir $primary $ceph_osd_args || return 1
698 activate_osd $dir $other $ceph_osd_args || return 1
699 wait_for_clean || return 1
700
701 repair $pgid
702 wait_for_clean || return 1
703 ceph pg dump pgs
704 flush_pg_stats
705
706 # This should have caused 1 object to be repaired
707 ceph pg $pgid query | jq '.info.stats.stat_sum'
708 COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired')
709 test "$COUNT" = "$REPAIRS" || return 1
710
711 ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $primary )"
712 COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $primary ).num_shards_repaired")
713 test "$COUNT" = "$(expr $REPAIRS / 2)" || return 1
714
715 ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $other )"
716 COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $other ).num_shards_repaired")
717 test "$COUNT" = "$(expr $REPAIRS / 2)" || return 1
718
719 ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum"
720 COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum.num_shards_repaired")
721 test "$COUNT" = "$REPAIRS" || return 1
722 }
723
724 function TEST_repair_stats_ec() {
725 local dir=$1
726 local poolname=testpool
727 local OSDS=3
728 local OBJS=30
729 # This need to be an even number
730 local REPAIRS=26
731 local allow_overwrites=false
732
733 # Launch a cluster with 5 seconds scrub interval
734 run_mon $dir a || return 1
735 run_mgr $dir x || return 1
736 local ceph_osd_args="--osd_deep_scrub_randomize_ratio=0 \
737 --osd-scrub-interval-randomize-ratio=0"
738 for id in $(seq 0 $(expr $OSDS - 1)) ; do
739 run_osd $dir $id $ceph_osd_args || return 1
740 done
741
742 # Create an EC pool
743 create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
744
745 # Put an object
746 local payload=ABCDEF
747 echo $payload > $dir/ORIGINAL
748 for i in $(seq 1 $OBJS)
749 do
750 rados --pool $poolname put obj$i $dir/ORIGINAL || return 1
751 done
752
753 # Remove the object from one shard physically
754 # Restarted osd get $ceph_osd_args passed
755 local other=$(get_not_primary $poolname obj1)
756 local pgid=$(get_pg $poolname obj1)
757 local primary=$(get_primary $poolname obj1)
758
759 kill_daemons $dir TERM osd.$other >&2 < /dev/null || return 1
760 kill_daemons $dir TERM osd.$primary >&2 < /dev/null || return 1
761 for i in $(seq 1 $REPAIRS)
762 do
763 # Remove from both osd.0 and osd.1
764 OSD=$(expr $i % 2)
765 _objectstore_tool_nodown $dir $OSD obj$i remove || return 1
766 done
767 activate_osd $dir $primary $ceph_osd_args || return 1
768 activate_osd $dir $other $ceph_osd_args || return 1
769 wait_for_clean || return 1
770
771 repair $pgid
772 wait_for_clean || return 1
773 ceph pg dump pgs
774 flush_pg_stats
775
776 # This should have caused 1 object to be repaired
777 ceph pg $pgid query | jq '.info.stats.stat_sum'
778 COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired')
779 test "$COUNT" = "$REPAIRS" || return 1
780
781 for osd in $(seq 0 $(expr $OSDS - 1)) ; do
782 if [ $osd = $other -o $osd = $primary ]; then
783 repair=$(expr $REPAIRS / 2)
784 else
785 repair="0"
786 fi
787
788 ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $osd )"
789 COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats[] | select(.osd == $osd ).num_shards_repaired")
790 test "$COUNT" = "$repair" || return 1
791 done
792
793 ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum"
794 COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum.num_shards_repaired")
795 test "$COUNT" = "$REPAIRS" || return 1
796 }
797
798 function corrupt_and_repair_jerasure() {
799 local dir=$1
800 local allow_overwrites=$2
801 local poolname=ecpool
802
803 run_mon $dir a || return 1
804 run_mgr $dir x || return 1
805 for id in $(seq 0 3) ; do
806 if [ "$allow_overwrites" = "true" ]; then
807 run_osd $dir $id || return 1
808 else
809 run_osd_filestore $dir $id || return 1
810 fi
811 done
812 create_rbd_pool || return 1
813 wait_for_clean || return 1
814
815 create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
816 corrupt_and_repair_erasure_coded $dir $poolname || return 1
817 }
818
819 function TEST_corrupt_and_repair_jerasure_appends() {
820 corrupt_and_repair_jerasure $1 false
821 }
822
823 function TEST_corrupt_and_repair_jerasure_overwrites() {
824 if [ "$use_ec_overwrite" = "true" ]; then
825 corrupt_and_repair_jerasure $1 true
826 fi
827 }
828
829 function corrupt_and_repair_lrc() {
830 local dir=$1
831 local allow_overwrites=$2
832 local poolname=ecpool
833
834 run_mon $dir a || return 1
835 run_mgr $dir x || return 1
836 for id in $(seq 0 9) ; do
837 if [ "$allow_overwrites" = "true" ]; then
838 run_osd $dir $id || return 1
839 else
840 run_osd_filestore $dir $id || return 1
841 fi
842 done
843 create_rbd_pool || return 1
844 wait_for_clean || return 1
845
846 create_ec_pool $poolname $allow_overwrites k=4 m=2 l=3 plugin=lrc || return 1
847 corrupt_and_repair_erasure_coded $dir $poolname || return 1
848 }
849
850 function TEST_corrupt_and_repair_lrc_appends() {
851 corrupt_and_repair_lrc $1 false
852 }
853
854 function TEST_corrupt_and_repair_lrc_overwrites() {
855 if [ "$use_ec_overwrite" = "true" ]; then
856 corrupt_and_repair_lrc $1 true
857 fi
858 }
859
860 function unfound_erasure_coded() {
861 local dir=$1
862 local allow_overwrites=$2
863 local poolname=ecpool
864 local payload=ABCDEF
865
866 run_mon $dir a || return 1
867 run_mgr $dir x || return 1
868 for id in $(seq 0 3) ; do
869 if [ "$allow_overwrites" = "true" ]; then
870 run_osd $dir $id || return 1
871 else
872 run_osd_filestore $dir $id || return 1
873 fi
874 done
875
876 create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1
877
878 add_something $dir $poolname || return 1
879
880 local primary=$(get_primary $poolname SOMETHING)
881 local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//"))
882 local not_primary_first=${osds[0]}
883 local not_primary_second=${osds[1]}
884 local not_primary_third=${osds[2]}
885
886 #
887 # 1) remove the corresponding file from the OSDs
888 #
889 pids=""
890 run_in_background pids objectstore_tool $dir $not_primary_first SOMETHING remove
891 run_in_background pids objectstore_tool $dir $not_primary_second SOMETHING remove
892 run_in_background pids objectstore_tool $dir $not_primary_third SOMETHING remove
893 wait_background pids
894 return_code=$?
895 if [ $return_code -ne 0 ]; then return $return_code; fi
896
897 #
898 # 2) repair the PG
899 #
900 local pg=$(get_pg $poolname SOMETHING)
901 repair $pg
902 #
903 # 3) check pg state
904 #
905 # it may take a bit to appear due to mon/mgr asynchrony
906 for f in `seq 1 60`; do
907 ceph -s | grep "1/1 objects unfound" && break
908 sleep 1
909 done
910 ceph -s|grep "4 up" || return 1
911 ceph -s|grep "4 in" || return 1
912 ceph -s|grep "1/1 objects unfound" || return 1
913 }
914
915 function TEST_unfound_erasure_coded_appends() {
916 unfound_erasure_coded $1 false
917 }
918
919 function TEST_unfound_erasure_coded_overwrites() {
920 if [ "$use_ec_overwrite" = "true" ]; then
921 unfound_erasure_coded $1 true
922 fi
923 }
924
925 #
926 # list_missing for EC pool
927 #
928 function list_missing_erasure_coded() {
929 local dir=$1
930 local allow_overwrites=$2
931 local poolname=ecpool
932
933 run_mon $dir a || return 1
934 run_mgr $dir x || return 1
935 for id in $(seq 0 2) ; do
936 if [ "$allow_overwrites" = "true" ]; then
937 run_osd $dir $id || return 1
938 else
939 run_osd_filestore $dir $id || return 1
940 fi
941 done
942 create_rbd_pool || return 1
943 wait_for_clean || return 1
944
945 create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1
946
947 # Put an object and remove the two shards (including primary)
948 add_something $dir $poolname MOBJ0 || return 1
949 local -a osds0=($(get_osds $poolname MOBJ0))
950
951 # Put another object and remove two shards (excluding primary)
952 add_something $dir $poolname MOBJ1 || return 1
953 local -a osds1=($(get_osds $poolname MOBJ1))
954
955 # Stop all osd daemons
956 for id in $(seq 0 2) ; do
957 kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1
958 done
959
960 id=${osds0[0]}
961 ceph-objectstore-tool --data-path $dir/$id \
962 MOBJ0 remove || return 1
963 id=${osds0[1]}
964 ceph-objectstore-tool --data-path $dir/$id \
965 MOBJ0 remove || return 1
966
967 id=${osds1[1]}
968 ceph-objectstore-tool --data-path $dir/$id \
969 MOBJ1 remove || return 1
970 id=${osds1[2]}
971 ceph-objectstore-tool --data-path $dir/$id \
972 MOBJ1 remove || return 1
973
974 for id in $(seq 0 2) ; do
975 activate_osd $dir $id >&2 || return 1
976 done
977 create_rbd_pool || return 1
978 wait_for_clean || return 1
979
980 # Get get - both objects should in the same PG
981 local pg=$(get_pg $poolname MOBJ0)
982
983 # Repair the PG, which triggers the recovering,
984 # and should mark the object as unfound
985 repair $pg
986
987 for i in $(seq 0 120) ; do
988 [ $i -lt 60 ] || return 1
989 matches=$(ceph pg $pg list_unfound | egrep "MOBJ0|MOBJ1" | wc -l)
990 [ $matches -eq 2 ] && break
991 done
992 }
993
994 function TEST_list_missing_erasure_coded_appends() {
995 list_missing_erasure_coded $1 false
996 }
997
998 function TEST_list_missing_erasure_coded_overwrites() {
999 if [ "$use_ec_overwrite" = "true" ]; then
1000 list_missing_erasure_coded $1 true
1001 fi
1002 }
1003
1004 #
1005 # Corrupt one copy of a replicated pool
1006 #
1007 function TEST_corrupt_scrub_replicated() {
1008 local dir=$1
1009 local poolname=csr_pool
1010 local total_objs=19
1011
1012 run_mon $dir a --osd_pool_default_size=2 || return 1
1013 run_mgr $dir x || return 1
1014 run_osd $dir 0 || return 1
1015 run_osd $dir 1 || return 1
1016 create_rbd_pool || return 1
1017 wait_for_clean || return 1
1018
1019 create_pool foo 1 || return 1
1020 create_pool $poolname 1 1 || return 1
1021 wait_for_clean || return 1
1022
1023 for i in $(seq 1 $total_objs) ; do
1024 objname=ROBJ${i}
1025 add_something $dir $poolname $objname || return 1
1026
1027 rados --pool $poolname setomapheader $objname hdr-$objname || return 1
1028 rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1
1029 done
1030
1031 # Increase file 1 MB + 1KB
1032 dd if=/dev/zero of=$dir/new.ROBJ19 bs=1024 count=1025
1033 rados --pool $poolname put $objname $dir/new.ROBJ19 || return 1
1034 rm -f $dir/new.ROBJ19
1035
1036 local pg=$(get_pg $poolname ROBJ0)
1037 local primary=$(get_primary $poolname ROBJ0)
1038
1039 # Compute an old omap digest and save oi
1040 CEPH_ARGS='' ceph daemon $(get_asok_path osd.0) \
1041 config set osd_deep_scrub_update_digest_min_age 0
1042 CEPH_ARGS='' ceph daemon $(get_asok_path osd.1) \
1043 config set osd_deep_scrub_update_digest_min_age 0
1044 pg_deep_scrub $pg
1045
1046 for i in $(seq 1 $total_objs) ; do
1047 objname=ROBJ${i}
1048
1049 # Alternate corruption between osd.0 and osd.1
1050 local osd=$(expr $i % 2)
1051
1052 case $i in
1053 1)
1054 # Size (deep scrub data_digest too)
1055 local payload=UVWXYZZZ
1056 echo $payload > $dir/CORRUPT
1057 objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
1058 ;;
1059
1060 2)
1061 # digest (deep scrub only)
1062 local payload=UVWXYZ
1063 echo $payload > $dir/CORRUPT
1064 objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
1065 ;;
1066
1067 3)
1068 # missing
1069 objectstore_tool $dir $osd $objname remove || return 1
1070 ;;
1071
1072 4)
1073 # Modify omap value (deep scrub only)
1074 objectstore_tool $dir $osd $objname set-omap key-$objname $dir/CORRUPT || return 1
1075 ;;
1076
1077 5)
1078 # Delete omap key (deep scrub only)
1079 objectstore_tool $dir $osd $objname rm-omap key-$objname || return 1
1080 ;;
1081
1082 6)
1083 # Add extra omap key (deep scrub only)
1084 echo extra > $dir/extra-val
1085 objectstore_tool $dir $osd $objname set-omap key2-$objname $dir/extra-val || return 1
1086 rm $dir/extra-val
1087 ;;
1088
1089 7)
1090 # Modify omap header (deep scrub only)
1091 echo -n newheader > $dir/hdr
1092 objectstore_tool $dir $osd $objname set-omaphdr $dir/hdr || return 1
1093 rm $dir/hdr
1094 ;;
1095
1096 8)
1097 rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
1098 rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
1099
1100 # Break xattrs
1101 echo -n bad-val > $dir/bad-val
1102 objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
1103 objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
1104 echo -n val3-$objname > $dir/newval
1105 objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
1106 rm $dir/bad-val $dir/newval
1107 ;;
1108
1109 9)
1110 objectstore_tool $dir $osd $objname get-attr _ > $dir/robj9-oi
1111 echo -n D > $dir/change
1112 rados --pool $poolname put $objname $dir/change
1113 objectstore_tool $dir $osd $objname set-attr _ $dir/robj9-oi
1114 rm $dir/oi $dir/change
1115 ;;
1116
1117 # ROBJ10 must be handled after digests are re-computed by a deep scrub below
1118 # ROBJ11 must be handled with config change before deep scrub
1119 # ROBJ12 must be handled with config change before scrubs
1120 # ROBJ13 must be handled before scrubs
1121
1122 14)
1123 echo -n bad-val > $dir/bad-val
1124 objectstore_tool $dir 0 $objname set-attr _ $dir/bad-val || return 1
1125 objectstore_tool $dir 1 $objname rm-attr _ || return 1
1126 rm $dir/bad-val
1127 ;;
1128
1129 15)
1130 objectstore_tool $dir $osd $objname rm-attr _ || return 1
1131 ;;
1132
1133 16)
1134 objectstore_tool $dir 0 $objname rm-attr snapset || return 1
1135 echo -n bad-val > $dir/bad-val
1136 objectstore_tool $dir 1 $objname set-attr snapset $dir/bad-val || return 1
1137 ;;
1138
1139 17)
1140 # Deep-scrub only (all replicas are diffent than the object info
1141 local payload=ROBJ17
1142 echo $payload > $dir/new.ROBJ17
1143 objectstore_tool $dir 0 $objname set-bytes $dir/new.ROBJ17 || return 1
1144 objectstore_tool $dir 1 $objname set-bytes $dir/new.ROBJ17 || return 1
1145 ;;
1146
1147 18)
1148 # Deep-scrub only (all replicas are diffent than the object info
1149 local payload=ROBJ18
1150 echo $payload > $dir/new.ROBJ18
1151 objectstore_tool $dir 0 $objname set-bytes $dir/new.ROBJ18 || return 1
1152 objectstore_tool $dir 1 $objname set-bytes $dir/new.ROBJ18 || return 1
1153 # Make one replica have a different object info, so a full repair must happen too
1154 objectstore_tool $dir $osd $objname corrupt-info || return 1
1155 ;;
1156
1157 19)
1158 # Set osd-max-object-size smaller than this object's size
1159
1160 esac
1161 done
1162
1163 local pg=$(get_pg $poolname ROBJ0)
1164
1165 ceph tell osd.\* injectargs -- --osd-max-object-size=1048576
1166
1167 inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
1168 inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
1169 inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
1170 inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
1171
1172 pg_scrub $pg
1173
1174 ERRORS=0
1175 declare -a s_err_strings
1176 err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:30259878:::ROBJ15:head : candidate had a missing info key"
1177 err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:33aca486:::ROBJ18:head : object info inconsistent "
1178 err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:5c7b2c47:::ROBJ16:head : candidate had a corrupt snapset"
1179 err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:5c7b2c47:::ROBJ16:head : candidate had a missing snapset key"
1180 err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:5c7b2c47:::ROBJ16:head : failed to pick suitable object info"
1181 err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:86586531:::ROBJ8:head : attr value mismatch '_key1-ROBJ8', attr name mismatch '_key3-ROBJ8', attr name mismatch '_key2-ROBJ8'"
1182 err_strings[6]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:bc819597:::ROBJ12:head : candidate had a stat error"
1183 err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:c0c86b1d:::ROBJ14:head : candidate had a missing info key"
1184 err_strings[8]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:c0c86b1d:::ROBJ14:head : candidate had a corrupt info"
1185 err_strings[9]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:c0c86b1d:::ROBJ14:head : failed to pick suitable object info"
1186 err_strings[10]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : candidate size 9 info size 7 mismatch"
1187 err_strings[11]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : size 9 != size 7 from auth oi 3:ce3f1d6a:::ROBJ1:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [[]0 0 0[]][)], size 9 != size 7 from shard 0"
1188 err_strings[12]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:d60617f9:::ROBJ13:head : candidate had a stat error"
1189 err_strings[13]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 3:f2a5b2a4:::ROBJ3:head : missing"
1190 err_strings[14]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ffdb2004:::ROBJ9:head : candidate size 1 info size 7 mismatch"
1191 err_strings[15]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ffdb2004:::ROBJ9:head : object info inconsistent "
1192 err_strings[16]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 3:c0c86b1d:::ROBJ14:head : no '_' attr"
1193 err_strings[17]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 3:5c7b2c47:::ROBJ16:head : can't decode 'snapset' attr .* no longer understand old encoding version 3 < 97: Malformed input"
1194 err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub : stat mismatch, got 19/19 objects, 0/0 clones, 18/19 dirty, 18/19 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 1049713/1049720 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes."
1195 err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 1 missing, 8 inconsistent objects"
1196 err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 18 errors"
1197 err_strings[21]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:123a5f55:::ROBJ19:head : size 1049600 > 1048576 is too large"
1198
1199 for err_string in "${err_strings[@]}"
1200 do
1201 if ! grep -q "$err_string" $dir/osd.${primary}.log
1202 then
1203 echo "Missing log message '$err_string'"
1204 ERRORS=$(expr $ERRORS + 1)
1205 fi
1206 done
1207
1208 rados list-inconsistent-pg $poolname > $dir/json || return 1
1209 # Check pg count
1210 test $(jq '. | length' $dir/json) = "1" || return 1
1211 # Check pgid
1212 test $(jq -r '.[0]' $dir/json) = $pg || return 1
1213
1214 rados list-inconsistent-obj $pg > $dir/json || return 1
1215 # Get epoch for repair-get requests
1216 epoch=$(jq .epoch $dir/json)
1217
1218 jq "$jqfilter" << EOF | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/checkcsjson
1219 {
1220 "inconsistents": [
1221 {
1222 "shards": [
1223 {
1224 "size": 7,
1225 "errors": [],
1226 "osd": 0,
1227 "primary": false
1228 },
1229 {
1230 "object_info": {
1231 "oid": {
1232 "oid": "ROBJ1",
1233 "key": "",
1234 "snapid": -2,
1235 "hash": 1454963827,
1236 "max": 0,
1237 "pool": 3,
1238 "namespace": ""
1239 },
1240 "version": "51'58",
1241 "prior_version": "21'3",
1242 "last_reqid": "osd.1.0:57",
1243 "user_version": 3,
1244 "size": 7,
1245 "mtime": "",
1246 "local_mtime": "",
1247 "lost": 0,
1248 "flags": [
1249 "dirty",
1250 "omap",
1251 "data_digest",
1252 "omap_digest"
1253 ],
1254 "truncate_seq": 0,
1255 "truncate_size": 0,
1256 "data_digest": "0x2ddbf8f5",
1257 "omap_digest": "0xf5fba2c6",
1258 "expected_object_size": 0,
1259 "expected_write_size": 0,
1260 "alloc_hint_flags": 0,
1261 "manifest": {
1262 "type": 0
1263 },
1264 "watchers": {}
1265 },
1266 "size": 9,
1267 "errors": [
1268 "size_mismatch_info",
1269 "obj_size_info_mismatch"
1270 ],
1271 "osd": 1,
1272 "primary": true
1273 }
1274 ],
1275 "selected_object_info": {
1276 "oid": {
1277 "oid": "ROBJ1",
1278 "key": "",
1279 "snapid": -2,
1280 "hash": 1454963827,
1281 "max": 0,
1282 "pool": 3,
1283 "namespace": ""
1284 },
1285 "version": "51'58",
1286 "prior_version": "21'3",
1287 "last_reqid": "osd.1.0:57",
1288 "user_version": 3,
1289 "size": 7,
1290 "mtime": "2018-04-05 14:33:19.804040",
1291 "local_mtime": "2018-04-05 14:33:19.804839",
1292 "lost": 0,
1293 "flags": [
1294 "dirty",
1295 "omap",
1296 "data_digest",
1297 "omap_digest"
1298 ],
1299 "truncate_seq": 0,
1300 "truncate_size": 0,
1301 "data_digest": "0x2ddbf8f5",
1302 "omap_digest": "0xf5fba2c6",
1303 "expected_object_size": 0,
1304 "expected_write_size": 0,
1305 "alloc_hint_flags": 0,
1306 "manifest": {
1307 "type": 0
1308 },
1309 "watchers": {}
1310 },
1311 "union_shard_errors": [
1312 "size_mismatch_info",
1313 "obj_size_info_mismatch"
1314 ],
1315 "errors": [
1316 "size_mismatch"
1317 ],
1318 "object": {
1319 "version": 3,
1320 "snap": "head",
1321 "locator": "",
1322 "nspace": "",
1323 "name": "ROBJ1"
1324 }
1325 },
1326 {
1327 "shards": [
1328 {
1329 "errors": [
1330 "stat_error"
1331 ],
1332 "osd": 0,
1333 "primary": false
1334 },
1335 {
1336 "size": 7,
1337 "errors": [],
1338 "osd": 1,
1339 "primary": true
1340 }
1341 ],
1342 "selected_object_info": {
1343 "oid": {
1344 "oid": "ROBJ12",
1345 "key": "",
1346 "snapid": -2,
1347 "hash": 3920199997,
1348 "max": 0,
1349 "pool": 3,
1350 "namespace": ""
1351 },
1352 "version": "51'56",
1353 "prior_version": "43'36",
1354 "last_reqid": "osd.1.0:55",
1355 "user_version": 36,
1356 "size": 7,
1357 "mtime": "",
1358 "local_mtime": "",
1359 "lost": 0,
1360 "flags": [
1361 "dirty",
1362 "omap",
1363 "data_digest",
1364 "omap_digest"
1365 ],
1366 "truncate_seq": 0,
1367 "truncate_size": 0,
1368 "data_digest": "0x2ddbf8f5",
1369 "omap_digest": "0x067f306a",
1370 "expected_object_size": 0,
1371 "expected_write_size": 0,
1372 "alloc_hint_flags": 0,
1373 "manifest": {
1374 "type": 0
1375 },
1376 "watchers": {}
1377 },
1378 "union_shard_errors": [
1379 "stat_error"
1380 ],
1381 "errors": [],
1382 "object": {
1383 "version": 36,
1384 "snap": "head",
1385 "locator": "",
1386 "nspace": "",
1387 "name": "ROBJ12"
1388 }
1389 },
1390 {
1391 "shards": [
1392 {
1393 "errors": [
1394 "stat_error"
1395 ],
1396 "osd": 0,
1397 "primary": false
1398 },
1399 {
1400 "size": 7,
1401 "errors": [],
1402 "osd": 1,
1403 "primary": true
1404 }
1405 ],
1406 "selected_object_info": {
1407 "oid": {
1408 "oid": "ROBJ13",
1409 "key": "",
1410 "snapid": -2,
1411 "hash": 2682806379,
1412 "max": 0,
1413 "pool": 3,
1414 "namespace": ""
1415 },
1416 "version": "51'59",
1417 "prior_version": "45'39",
1418 "last_reqid": "osd.1.0:58",
1419 "user_version": 39,
1420 "size": 7,
1421 "mtime": "",
1422 "local_mtime": "",
1423 "lost": 0,
1424 "flags": [
1425 "dirty",
1426 "omap",
1427 "data_digest",
1428 "omap_digest"
1429 ],
1430 "truncate_seq": 0,
1431 "truncate_size": 0,
1432 "data_digest": "0x2ddbf8f5",
1433 "omap_digest": "0x6441854d",
1434 "expected_object_size": 0,
1435 "expected_write_size": 0,
1436 "alloc_hint_flags": 0,
1437 "manifest": {
1438 "type": 0
1439 },
1440 "watchers": {}
1441 },
1442 "union_shard_errors": [
1443 "stat_error"
1444 ],
1445 "errors": [],
1446 "object": {
1447 "version": 39,
1448 "snap": "head",
1449 "locator": "",
1450 "nspace": "",
1451 "name": "ROBJ13"
1452 }
1453 },
1454 {
1455 "shards": [
1456 {
1457 "object_info": "bad-val",
1458 "size": 7,
1459 "errors": [
1460 "info_corrupted"
1461 ],
1462 "osd": 0,
1463 "primary": false
1464 },
1465 {
1466 "size": 7,
1467 "errors": [
1468 "info_missing"
1469 ],
1470 "osd": 1,
1471 "primary": true
1472 }
1473 ],
1474 "union_shard_errors": [
1475 "info_missing",
1476 "info_corrupted"
1477 ],
1478 "errors": [],
1479 "object": {
1480 "version": 0,
1481 "snap": "head",
1482 "locator": "",
1483 "nspace": "",
1484 "name": "ROBJ14"
1485 }
1486 },
1487 {
1488 "shards": [
1489 {
1490 "object_info": {
1491 "oid": {
1492 "oid": "ROBJ15",
1493 "key": "",
1494 "snapid": -2,
1495 "hash": 504996876,
1496 "max": 0,
1497 "pool": 3,
1498 "namespace": ""
1499 },
1500 "version": "51'49",
1501 "prior_version": "49'45",
1502 "last_reqid": "osd.1.0:48",
1503 "user_version": 45,
1504 "size": 7,
1505 "mtime": "2018-04-05 14:33:29.498969",
1506 "local_mtime": "2018-04-05 14:33:29.499890",
1507 "lost": 0,
1508 "flags": [
1509 "dirty",
1510 "omap",
1511 "data_digest",
1512 "omap_digest"
1513 ],
1514 "truncate_seq": 0,
1515 "truncate_size": 0,
1516 "data_digest": "0x2ddbf8f5",
1517 "omap_digest": "0x2d2a4d6e",
1518 "expected_object_size": 0,
1519 "expected_write_size": 0,
1520 "alloc_hint_flags": 0,
1521 "manifest": {
1522 "type": 0
1523 },
1524 "watchers": {}
1525 },
1526 "size": 7,
1527 "errors": [],
1528 "osd": 0,
1529 "primary": false
1530 },
1531 {
1532 "size": 7,
1533 "errors": [
1534 "info_missing"
1535 ],
1536 "osd": 1,
1537 "primary": true
1538 }
1539 ],
1540 "selected_object_info": {
1541 "oid": {
1542 "oid": "ROBJ15",
1543 "key": "",
1544 "snapid": -2,
1545 "hash": 504996876,
1546 "max": 0,
1547 "pool": 3,
1548 "namespace": ""
1549 },
1550 "version": "51'49",
1551 "prior_version": "49'45",
1552 "last_reqid": "osd.1.0:48",
1553 "user_version": 45,
1554 "size": 7,
1555 "mtime": "",
1556 "local_mtime": "",
1557 "lost": 0,
1558 "flags": [
1559 "dirty",
1560 "omap",
1561 "data_digest",
1562 "omap_digest"
1563 ],
1564 "truncate_seq": 0,
1565 "truncate_size": 0,
1566 "data_digest": "0x2ddbf8f5",
1567 "omap_digest": "0x2d2a4d6e",
1568 "expected_object_size": 0,
1569 "expected_write_size": 0,
1570 "alloc_hint_flags": 0,
1571 "manifest": {
1572 "type": 0
1573 },
1574 "watchers": {}
1575 },
1576 "union_shard_errors": [
1577 "info_missing"
1578 ],
1579 "errors": [],
1580 "object": {
1581 "version": 45,
1582 "snap": "head",
1583 "locator": "",
1584 "nspace": "",
1585 "name": "ROBJ15"
1586 }
1587 },
1588 {
1589 "errors": [],
1590 "object": {
1591 "locator": "",
1592 "name": "ROBJ16",
1593 "nspace": "",
1594 "snap": "head",
1595 "version": 0
1596 },
1597 "shards": [
1598 {
1599 "errors": [
1600 "snapset_missing"
1601 ],
1602 "osd": 0,
1603 "primary": false,
1604 "size": 7
1605 },
1606 {
1607 "errors": [
1608 "snapset_corrupted"
1609 ],
1610 "osd": 1,
1611 "primary": true,
1612 "snapset": "bad-val",
1613 "size": 7
1614 }
1615 ],
1616 "union_shard_errors": [
1617 "snapset_missing",
1618 "snapset_corrupted"
1619 ]
1620 },
1621 {
1622 "errors": [
1623 "object_info_inconsistency"
1624 ],
1625 "object": {
1626 "locator": "",
1627 "name": "ROBJ18",
1628 "nspace": "",
1629 "snap": "head"
1630 },
1631 "selected_object_info": {
1632 "alloc_hint_flags": 255,
1633 "data_digest": "0x2ddbf8f5",
1634 "expected_object_size": 0,
1635 "expected_write_size": 0,
1636 "flags": [
1637 "dirty",
1638 "omap",
1639 "data_digest",
1640 "omap_digest"
1641 ],
1642 "lost": 0,
1643 "manifest": {
1644 "type": 0
1645 },
1646 "oid": {
1647 "hash": 1629828556,
1648 "key": "",
1649 "max": 0,
1650 "namespace": "",
1651 "oid": "ROBJ18",
1652 "pool": 3,
1653 "snapid": -2
1654 },
1655 "omap_digest": "0xddc3680f",
1656 "size": 7,
1657 "truncate_seq": 0,
1658 "truncate_size": 0,
1659 "user_version": 54,
1660 "watchers": {}
1661 },
1662 "shards": [
1663 {
1664 "errors": [],
1665 "object_info": {
1666 "alloc_hint_flags": 0,
1667 "data_digest": "0x2ddbf8f5",
1668 "expected_object_size": 0,
1669 "expected_write_size": 0,
1670 "flags": [
1671 "dirty",
1672 "omap",
1673 "data_digest",
1674 "omap_digest"
1675 ],
1676 "lost": 0,
1677 "manifest": {
1678 "type": 0
1679 },
1680 "oid": {
1681 "hash": 1629828556,
1682 "key": "",
1683 "max": 0,
1684 "namespace": "",
1685 "oid": "ROBJ18",
1686 "pool": 3,
1687 "snapid": -2
1688 },
1689 "omap_digest": "0xddc3680f",
1690 "size": 7,
1691 "truncate_seq": 0,
1692 "truncate_size": 0,
1693 "user_version": 54,
1694 "watchers": {}
1695 },
1696 "osd": 0,
1697 "primary": false,
1698 "size": 7
1699 },
1700 {
1701 "errors": [],
1702 "object_info": {
1703 "alloc_hint_flags": 255,
1704 "data_digest": "0x2ddbf8f5",
1705 "expected_object_size": 0,
1706 "expected_write_size": 0,
1707 "flags": [
1708 "dirty",
1709 "omap",
1710 "data_digest",
1711 "omap_digest"
1712 ],
1713 "lost": 0,
1714 "manifest": {
1715 "type": 0
1716 },
1717 "oid": {
1718 "hash": 1629828556,
1719 "key": "",
1720 "max": 0,
1721 "namespace": "",
1722 "oid": "ROBJ18",
1723 "pool": 3,
1724 "snapid": -2
1725 },
1726 "omap_digest": "0xddc3680f",
1727 "size": 7,
1728 "truncate_seq": 0,
1729 "truncate_size": 0,
1730 "user_version": 54,
1731 "watchers": {}
1732 },
1733 "osd": 1,
1734 "primary": true,
1735 "size": 7
1736 }
1737 ],
1738 "union_shard_errors": []
1739 },
1740 {
1741 "object": {
1742 "name": "ROBJ19",
1743 "nspace": "",
1744 "locator": "",
1745 "snap": "head",
1746 "version": 58
1747 },
1748 "errors": [
1749 "size_too_large"
1750 ],
1751 "union_shard_errors": [],
1752 "selected_object_info": {
1753 "oid": {
1754 "oid": "ROBJ19",
1755 "key": "",
1756 "snapid": -2,
1757 "hash": 2868534344,
1758 "max": 0,
1759 "pool": 3,
1760 "namespace": ""
1761 },
1762 "version": "63'59",
1763 "prior_version": "63'58",
1764 "last_reqid": "osd.1.0:58",
1765 "user_version": 58,
1766 "size": 1049600,
1767 "mtime": "2019-08-09T23:33:58.340709+0000",
1768 "local_mtime": "2019-08-09T23:33:58.345676+0000",
1769 "lost": 0,
1770 "flags": [
1771 "dirty",
1772 "omap",
1773 "data_digest",
1774 "omap_digest"
1775 ],
1776 "truncate_seq": 0,
1777 "truncate_size": 0,
1778 "data_digest": "0x3dde0ef3",
1779 "omap_digest": "0xbffddd28",
1780 "expected_object_size": 0,
1781 "expected_write_size": 0,
1782 "alloc_hint_flags": 0,
1783 "manifest": {
1784 "type": 0
1785 },
1786 "watchers": {}
1787 },
1788 "shards": [
1789 {
1790 "osd": 0,
1791 "primary": false,
1792 "errors": [],
1793 "size": 1049600
1794 },
1795 {
1796 "osd": 1,
1797 "primary": true,
1798 "errors": [],
1799 "size": 1049600
1800 }
1801 ]
1802 },
1803 {
1804 "shards": [
1805 {
1806 "size": 7,
1807 "errors": [],
1808 "osd": 0,
1809 "primary": false
1810 },
1811 {
1812 "errors": [
1813 "missing"
1814 ],
1815 "osd": 1,
1816 "primary": true
1817 }
1818 ],
1819 "selected_object_info": {
1820 "oid": {
1821 "oid": "ROBJ3",
1822 "key": "",
1823 "snapid": -2,
1824 "hash": 625845583,
1825 "max": 0,
1826 "pool": 3,
1827 "namespace": ""
1828 },
1829 "version": "51'61",
1830 "prior_version": "25'9",
1831 "last_reqid": "osd.1.0:60",
1832 "user_version": 9,
1833 "size": 7,
1834 "mtime": "",
1835 "local_mtime": "",
1836 "lost": 0,
1837 "flags": [
1838 "dirty",
1839 "omap",
1840 "data_digest",
1841 "omap_digest"
1842 ],
1843 "truncate_seq": 0,
1844 "truncate_size": 0,
1845 "data_digest": "0x2ddbf8f5",
1846 "omap_digest": "0x00b35dfd",
1847 "expected_object_size": 0,
1848 "expected_write_size": 0,
1849 "alloc_hint_flags": 0,
1850 "manifest": {
1851 "type": 0
1852 },
1853 "watchers": {}
1854 },
1855 "union_shard_errors": [
1856 "missing"
1857 ],
1858 "errors": [],
1859 "object": {
1860 "version": 9,
1861 "snap": "head",
1862 "locator": "",
1863 "nspace": "",
1864 "name": "ROBJ3"
1865 }
1866 },
1867 {
1868 "shards": [
1869 {
1870 "attrs": [
1871 {
1872 "Base64": false,
1873 "value": "bad-val",
1874 "name": "key1-ROBJ8"
1875 },
1876 {
1877 "Base64": false,
1878 "value": "val2-ROBJ8",
1879 "name": "key2-ROBJ8"
1880 }
1881 ],
1882 "size": 7,
1883 "errors": [],
1884 "osd": 0,
1885 "primary": false
1886 },
1887 {
1888 "attrs": [
1889 {
1890 "Base64": false,
1891 "value": "val1-ROBJ8",
1892 "name": "key1-ROBJ8"
1893 },
1894 {
1895 "Base64": false,
1896 "value": "val3-ROBJ8",
1897 "name": "key3-ROBJ8"
1898 }
1899 ],
1900 "size": 7,
1901 "errors": [],
1902 "osd": 1,
1903 "primary": true
1904 }
1905 ],
1906 "selected_object_info": {
1907 "oid": {
1908 "oid": "ROBJ8",
1909 "key": "",
1910 "snapid": -2,
1911 "hash": 2359695969,
1912 "max": 0,
1913 "pool": 3,
1914 "namespace": ""
1915 },
1916 "version": "79'66",
1917 "prior_version": "79'65",
1918 "last_reqid": "client.4554.0:1",
1919 "user_version": 79,
1920 "size": 7,
1921 "mtime": "",
1922 "local_mtime": "",
1923 "lost": 0,
1924 "flags": [
1925 "dirty",
1926 "omap",
1927 "data_digest",
1928 "omap_digest"
1929 ],
1930 "truncate_seq": 0,
1931 "truncate_size": 0,
1932 "data_digest": "0x2ddbf8f5",
1933 "omap_digest": "0xd6be81dc",
1934 "expected_object_size": 0,
1935 "expected_write_size": 0,
1936 "alloc_hint_flags": 0,
1937 "manifest": {
1938 "type": 0
1939 },
1940 "watchers": {}
1941 },
1942 "union_shard_errors": [],
1943 "errors": [
1944 "attr_value_mismatch",
1945 "attr_name_mismatch"
1946 ],
1947 "object": {
1948 "version": 66,
1949 "snap": "head",
1950 "locator": "",
1951 "nspace": "",
1952 "name": "ROBJ8"
1953 }
1954 },
1955 {
1956 "shards": [
1957 {
1958 "object_info": {
1959 "oid": {
1960 "oid": "ROBJ9",
1961 "key": "",
1962 "snapid": -2,
1963 "hash": 537189375,
1964 "max": 0,
1965 "pool": 3,
1966 "namespace": ""
1967 },
1968 "version": "95'67",
1969 "prior_version": "51'64",
1970 "last_reqid": "client.4649.0:1",
1971 "user_version": 80,
1972 "size": 1,
1973 "mtime": "",
1974 "local_mtime": "",
1975 "lost": 0,
1976 "flags": [
1977 "dirty",
1978 "omap",
1979 "data_digest",
1980 "omap_digest"
1981 ],
1982 "truncate_seq": 0,
1983 "truncate_size": 0,
1984 "data_digest": "0x2b63260d",
1985 "omap_digest": "0x2eecc539",
1986 "expected_object_size": 0,
1987 "expected_write_size": 0,
1988 "alloc_hint_flags": 0,
1989 "manifest": {
1990 "type": 0
1991 },
1992 "watchers": {}
1993 },
1994 "size": 1,
1995 "errors": [],
1996 "osd": 0,
1997 "primary": false
1998 },
1999 {
2000 "object_info": {
2001 "oid": {
2002 "oid": "ROBJ9",
2003 "key": "",
2004 "snapid": -2,
2005 "hash": 537189375,
2006 "max": 0,
2007 "pool": 3,
2008 "namespace": ""
2009 },
2010 "version": "51'64",
2011 "prior_version": "37'27",
2012 "last_reqid": "osd.1.0:63",
2013 "user_version": 27,
2014 "size": 7,
2015 "mtime": "2018-04-05 14:33:25.352485",
2016 "local_mtime": "2018-04-05 14:33:25.353746",
2017 "lost": 0,
2018 "flags": [
2019 "dirty",
2020 "omap",
2021 "data_digest",
2022 "omap_digest"
2023 ],
2024 "truncate_seq": 0,
2025 "truncate_size": 0,
2026 "data_digest": "0x2ddbf8f5",
2027 "omap_digest": "0x2eecc539",
2028 "expected_object_size": 0,
2029 "expected_write_size": 0,
2030 "alloc_hint_flags": 0,
2031 "manifest": {
2032 "type": 0
2033 },
2034 "watchers": {}
2035 },
2036 "size": 1,
2037 "errors": [
2038 "obj_size_info_mismatch"
2039 ],
2040 "osd": 1,
2041 "primary": true
2042 }
2043 ],
2044 "selected_object_info": {
2045 "oid": {
2046 "oid": "ROBJ9",
2047 "key": "",
2048 "snapid": -2,
2049 "hash": 537189375,
2050 "max": 0,
2051 "pool": 3,
2052 "namespace": ""
2053 },
2054 "version": "95'67",
2055 "prior_version": "51'64",
2056 "last_reqid": "client.4649.0:1",
2057 "user_version": 80,
2058 "size": 1,
2059 "mtime": "",
2060 "local_mtime": "",
2061 "lost": 0,
2062 "flags": [
2063 "dirty",
2064 "omap",
2065 "data_digest",
2066 "omap_digest"
2067 ],
2068 "truncate_seq": 0,
2069 "truncate_size": 0,
2070 "data_digest": "0x2b63260d",
2071 "omap_digest": "0x2eecc539",
2072 "expected_object_size": 0,
2073 "expected_write_size": 0,
2074 "alloc_hint_flags": 0,
2075 "manifest": {
2076 "type": 0
2077 },
2078 "watchers": {}
2079 },
2080 "union_shard_errors": [
2081 "obj_size_info_mismatch"
2082 ],
2083 "errors": [
2084 "object_info_inconsistency"
2085 ],
2086 "object": {
2087 "version": 67,
2088 "snap": "head",
2089 "locator": "",
2090 "nspace": "",
2091 "name": "ROBJ9"
2092 }
2093 }
2094 ],
2095 "epoch": 0
2096 }
2097 EOF
2098
2099 jq "$jqfilter" $dir/json | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/csjson
2100 multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
2101 if test $getjson = "yes"
2102 then
2103 jq '.' $dir/json > save1.json
2104 fi
2105
2106 if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
2107 then
2108 jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
2109 fi
2110
2111 objname=ROBJ9
2112 # Change data and size again because digest was recomputed
2113 echo -n ZZZ > $dir/change
2114 rados --pool $poolname put $objname $dir/change
2115 # Set one to an even older value
2116 objectstore_tool $dir 0 $objname set-attr _ $dir/robj9-oi
2117 rm $dir/oi $dir/change
2118
2119 objname=ROBJ10
2120 objectstore_tool $dir 1 $objname get-attr _ > $dir/oi
2121 rados --pool $poolname setomapval $objname key2-$objname val2-$objname
2122 objectstore_tool $dir 0 $objname set-attr _ $dir/oi
2123 objectstore_tool $dir 1 $objname set-attr _ $dir/oi
2124 rm $dir/oi
2125
2126 inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
2127 inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
2128 inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0
2129 inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
2130
2131 # ROBJ19 won't error this time
2132 ceph tell osd.\* injectargs -- --osd-max-object-size=134217728
2133
2134 pg_deep_scrub $pg
2135
2136 err_strings=()
2137 err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:30259878:::ROBJ15:head : candidate had a missing info key"
2138 err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:33aca486:::ROBJ18:head : data_digest 0xbd89c912 != data_digest 0x2ddbf8f5 from auth oi 3:33aca486:::ROBJ18:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 54 dd 2ddbf8f5 od ddc3680f alloc_hint [[]0 0 255[]][)], object info inconsistent "
2139 err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:33aca486:::ROBJ18:head : data_digest 0xbd89c912 != data_digest 0x2ddbf8f5 from auth oi 3:33aca486:::ROBJ18:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 54 dd 2ddbf8f5 od ddc3680f alloc_hint [[]0 0 255[]][)]"
2140 err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:33aca486:::ROBJ18:head : failed to pick suitable auth object"
2141 err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:5c7b2c47:::ROBJ16:head : candidate had a corrupt snapset"
2142 err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:5c7b2c47:::ROBJ16:head : candidate had a missing snapset key"
2143 err_strings[6]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:5c7b2c47:::ROBJ16:head : failed to pick suitable object info"
2144 err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:86586531:::ROBJ8:head : attr value mismatch '_key1-ROBJ8', attr name mismatch '_key3-ROBJ8', attr name mismatch '_key2-ROBJ8'"
2145 err_strings[8]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:87abbf36:::ROBJ11:head : candidate had a read error"
2146 err_strings[9]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:8aa5320e:::ROBJ17:head : data_digest 0x5af0c3ef != data_digest 0x2ddbf8f5 from auth oi 3:8aa5320e:::ROBJ17:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 51 dd 2ddbf8f5 od e9572720 alloc_hint [[]0 0 0[]][)]"
2147 err_strings[10]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:8aa5320e:::ROBJ17:head : data_digest 0x5af0c3ef != data_digest 0x2ddbf8f5 from auth oi 3:8aa5320e:::ROBJ17:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 51 dd 2ddbf8f5 od e9572720 alloc_hint [[]0 0 0[]][)]"
2148 err_strings[11]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:8aa5320e:::ROBJ17:head : failed to pick suitable auth object"
2149 err_strings[12]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:8b55fa4b:::ROBJ7:head : omap_digest 0xefced57a != omap_digest 0x6a73cc07 from shard 1"
2150 err_strings[13]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:8b55fa4b:::ROBJ7:head : omap_digest 0x6a73cc07 != omap_digest 0xefced57a from auth oi 3:8b55fa4b:::ROBJ7:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 21 dd 2ddbf8f5 od efced57a alloc_hint [[]0 0 0[]][)]"
2151 err_strings[14]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:a53c12e8:::ROBJ6:head : omap_digest 0x689ee887 != omap_digest 0x179c919f from shard 1, omap_digest 0x689ee887 != omap_digest 0x179c919f from auth oi 3:a53c12e8:::ROBJ6:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 18 dd 2ddbf8f5 od 179c919f alloc_hint [[]0 0 0[]][)]"
2152 err_strings[15]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:b1f19cbd:::ROBJ10:head : omap_digest 0xa8dd5adc != omap_digest 0xc2025a24 from auth oi 3:b1f19cbd:::ROBJ10:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [[]0 0 0[]][)]"
2153 err_strings[16]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:b1f19cbd:::ROBJ10:head : omap_digest 0xa8dd5adc != omap_digest 0xc2025a24 from auth oi 3:b1f19cbd:::ROBJ10:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [[]0 0 0[]][)]"
2154 err_strings[17]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:b1f19cbd:::ROBJ10:head : failed to pick suitable auth object"
2155 err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:bc819597:::ROBJ12:head : candidate had a stat error"
2156 err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:c0c86b1d:::ROBJ14:head : candidate had a missing info key"
2157 err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:c0c86b1d:::ROBJ14:head : candidate had a corrupt info"
2158 err_strings[21]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:c0c86b1d:::ROBJ14:head : failed to pick suitable object info"
2159 err_strings[22]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : candidate size 9 info size 7 mismatch"
2160 err_strings[23]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : data_digest 0x2d4a11c2 != data_digest 0x2ddbf8f5 from shard 0, data_digest 0x2d4a11c2 != data_digest 0x2ddbf8f5 from auth oi 3:ce3f1d6a:::ROBJ1:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [[]0 0 0[]][)], size 9 != size 7 from auth oi 3:ce3f1d6a:::ROBJ1:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [[]0 0 0[]][)], size 9 != size 7 from shard 0"
2161 err_strings[24]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:d60617f9:::ROBJ13:head : candidate had a read error"
2162 err_strings[25]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:d60617f9:::ROBJ13:head : candidate had a stat error"
2163 err_strings[26]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:d60617f9:::ROBJ13:head : failed to pick suitable object info"
2164 err_strings[27]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:e97ce31e:::ROBJ2:head : data_digest 0x578a4830 != data_digest 0x2ddbf8f5 from shard 1, data_digest 0x578a4830 != data_digest 0x2ddbf8f5 from auth oi 3:e97ce31e:::ROBJ2:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od f8e11918 alloc_hint [[]0 0 0[]][)]"
2165 err_strings[28]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 3:f2a5b2a4:::ROBJ3:head : missing"
2166 err_strings[29]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:f4981d31:::ROBJ4:head : omap_digest 0xd7178dfe != omap_digest 0xe2d46ea4 from shard 1, omap_digest 0xd7178dfe != omap_digest 0xe2d46ea4 from auth oi 3:f4981d31:::ROBJ4:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 12 dd 2ddbf8f5 od e2d46ea4 alloc_hint [[]0 0 0[]][)]"
2167 err_strings[30]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:f4bfd4d1:::ROBJ5:head : omap_digest 0x1a862a41 != omap_digest 0x6cac8f6 from shard 1"
2168 err_strings[31]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:f4bfd4d1:::ROBJ5:head : omap_digest 0x6cac8f6 != omap_digest 0x1a862a41 from auth oi 3:f4bfd4d1:::ROBJ5:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 15 dd 2ddbf8f5 od 1a862a41 alloc_hint [[]0 0 0[]][)]"
2169 err_strings[32]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:ffdb2004:::ROBJ9:head : candidate size 3 info size 7 mismatch"
2170 err_strings[33]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:ffdb2004:::ROBJ9:head : object info inconsistent "
2171 err_strings[34]="log_channel[(]cluster[)] log [[]ERR[]] : deep-scrub [0-9]*[.]0 3:c0c86b1d:::ROBJ14:head : no '_' attr"
2172 err_strings[35]="log_channel[(]cluster[)] log [[]ERR[]] : deep-scrub [0-9]*[.]0 3:5c7b2c47:::ROBJ16:head : can't decode 'snapset' attr .* no longer understand old encoding version 3 < 97: Malformed input"
2173 err_strings[36]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 deep-scrub : stat mismatch, got 19/19 objects, 0/0 clones, 18/19 dirty, 18/19 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 1049715/1049716 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes."
2174 err_strings[37]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 deep-scrub 1 missing, 11 inconsistent objects"
2175 err_strings[38]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 deep-scrub 35 errors"
2176
2177 for err_string in "${err_strings[@]}"
2178 do
2179 if ! grep -q "$err_string" $dir/osd.${primary}.log
2180 then
2181 echo "Missing log message '$err_string'"
2182 ERRORS=$(expr $ERRORS + 1)
2183 fi
2184 done
2185
2186 rados list-inconsistent-pg $poolname > $dir/json || return 1
2187 # Check pg count
2188 test $(jq '. | length' $dir/json) = "1" || return 1
2189 # Check pgid
2190 test $(jq -r '.[0]' $dir/json) = $pg || return 1
2191
2192 rados list-inconsistent-obj $pg > $dir/json || return 1
2193 # Get epoch for repair-get requests
2194 epoch=$(jq .epoch $dir/json)
2195
2196 jq "$jqfilter" << EOF | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/checkcsjson
2197 {
2198 "inconsistents": [
2199 {
2200 "shards": [
2201 {
2202 "data_digest": "0x2ddbf8f5",
2203 "omap_digest": "0xf5fba2c6",
2204 "size": 7,
2205 "errors": [],
2206 "osd": 0,
2207 "primary": false
2208 },
2209 {
2210 "object_info": {
2211 "oid": {
2212 "oid": "ROBJ1",
2213 "key": "",
2214 "snapid": -2,
2215 "hash": 1454963827,
2216 "max": 0,
2217 "pool": 3,
2218 "namespace": ""
2219 },
2220 "version": "51'58",
2221 "prior_version": "21'3",
2222 "last_reqid": "osd.1.0:57",
2223 "user_version": 3,
2224 "size": 7,
2225 "mtime": "2018-04-05 14:33:19.804040",
2226 "local_mtime": "2018-04-05 14:33:19.804839",
2227 "lost": 0,
2228 "flags": [
2229 "dirty",
2230 "omap",
2231 "data_digest",
2232 "omap_digest"
2233 ],
2234 "truncate_seq": 0,
2235 "truncate_size": 0,
2236 "data_digest": "0x2ddbf8f5",
2237 "omap_digest": "0xf5fba2c6",
2238 "expected_object_size": 0,
2239 "expected_write_size": 0,
2240 "alloc_hint_flags": 0,
2241 "manifest": {
2242 "type": 0
2243 },
2244 "watchers": {}
2245 },
2246 "data_digest": "0x2d4a11c2",
2247 "omap_digest": "0xf5fba2c6",
2248 "size": 9,
2249 "errors": [
2250 "data_digest_mismatch_info",
2251 "size_mismatch_info",
2252 "obj_size_info_mismatch"
2253 ],
2254 "osd": 1,
2255 "primary": true
2256 }
2257 ],
2258 "selected_object_info": {
2259 "oid": {
2260 "oid": "ROBJ1",
2261 "key": "",
2262 "snapid": -2,
2263 "hash": 1454963827,
2264 "max": 0,
2265 "pool": 3,
2266 "namespace": ""
2267 },
2268 "version": "51'58",
2269 "prior_version": "21'3",
2270 "last_reqid": "osd.1.0:57",
2271 "user_version": 3,
2272 "size": 7,
2273 "mtime": "2018-04-05 14:33:19.804040",
2274 "local_mtime": "2018-04-05 14:33:19.804839",
2275 "lost": 0,
2276 "flags": [
2277 "dirty",
2278 "omap",
2279 "data_digest",
2280 "omap_digest"
2281 ],
2282 "truncate_seq": 0,
2283 "truncate_size": 0,
2284 "data_digest": "0x2ddbf8f5",
2285 "omap_digest": "0xf5fba2c6",
2286 "expected_object_size": 0,
2287 "expected_write_size": 0,
2288 "alloc_hint_flags": 0,
2289 "manifest": {
2290 "type": 0
2291 },
2292 "watchers": {}
2293 },
2294 "union_shard_errors": [
2295 "data_digest_mismatch_info",
2296 "size_mismatch_info",
2297 "obj_size_info_mismatch"
2298 ],
2299 "errors": [
2300 "data_digest_mismatch",
2301 "size_mismatch"
2302 ],
2303 "object": {
2304 "version": 3,
2305 "snap": "head",
2306 "locator": "",
2307 "nspace": "",
2308 "name": "ROBJ1"
2309 }
2310 },
2311 {
2312 "shards": [
2313 {
2314 "data_digest": "0x2ddbf8f5",
2315 "omap_digest": "0xa8dd5adc",
2316 "size": 7,
2317 "errors": [
2318 "omap_digest_mismatch_info"
2319 ],
2320 "osd": 0,
2321 "primary": false
2322 },
2323 {
2324 "data_digest": "0x2ddbf8f5",
2325 "omap_digest": "0xa8dd5adc",
2326 "size": 7,
2327 "errors": [
2328 "omap_digest_mismatch_info"
2329 ],
2330 "osd": 1,
2331 "primary": true
2332 }
2333 ],
2334 "selected_object_info": {
2335 "alloc_hint_flags": 0,
2336 "data_digest": "0x2ddbf8f5",
2337 "expected_object_size": 0,
2338 "expected_write_size": 0,
2339 "flags": [
2340 "dirty",
2341 "omap",
2342 "data_digest",
2343 "omap_digest"
2344 ],
2345 "lost": 0,
2346 "manifest": {
2347 "type": 0
2348 },
2349 "oid": {
2350 "hash": 3174666125,
2351 "key": "",
2352 "max": 0,
2353 "namespace": "",
2354 "oid": "ROBJ10",
2355 "pool": 3,
2356 "snapid": -2
2357 },
2358 "omap_digest": "0xc2025a24",
2359 "size": 7,
2360 "truncate_seq": 0,
2361 "truncate_size": 0,
2362 "user_version": 30,
2363 "watchers": {}
2364 },
2365 "union_shard_errors": [
2366 "omap_digest_mismatch_info"
2367 ],
2368 "errors": [],
2369 "object": {
2370 "version": 30,
2371 "snap": "head",
2372 "locator": "",
2373 "nspace": "",
2374 "name": "ROBJ10"
2375 }
2376 },
2377 {
2378 "shards": [
2379 {
2380 "data_digest": "0x2ddbf8f5",
2381 "omap_digest": "0xa03cef03",
2382 "size": 7,
2383 "errors": [],
2384 "osd": 0,
2385 "primary": false
2386 },
2387 {
2388 "size": 7,
2389 "errors": [
2390 "read_error"
2391 ],
2392 "osd": 1,
2393 "primary": true
2394 }
2395 ],
2396 "selected_object_info": {
2397 "oid": {
2398 "oid": "ROBJ11",
2399 "key": "",
2400 "snapid": -2,
2401 "hash": 1828574689,
2402 "max": 0,
2403 "pool": 3,
2404 "namespace": ""
2405 },
2406 "version": "51'52",
2407 "prior_version": "41'33",
2408 "last_reqid": "osd.1.0:51",
2409 "user_version": 33,
2410 "size": 7,
2411 "mtime": "2018-04-05 14:33:26.761286",
2412 "local_mtime": "2018-04-05 14:33:26.762368",
2413 "lost": 0,
2414 "flags": [
2415 "dirty",
2416 "omap",
2417 "data_digest",
2418 "omap_digest"
2419 ],
2420 "truncate_seq": 0,
2421 "truncate_size": 0,
2422 "data_digest": "0x2ddbf8f5",
2423 "omap_digest": "0xa03cef03",
2424 "expected_object_size": 0,
2425 "expected_write_size": 0,
2426 "alloc_hint_flags": 0,
2427 "manifest": {
2428 "type": 0
2429 },
2430 "watchers": {}
2431 },
2432 "union_shard_errors": [
2433 "read_error"
2434 ],
2435 "errors": [],
2436 "object": {
2437 "version": 33,
2438 "snap": "head",
2439 "locator": "",
2440 "nspace": "",
2441 "name": "ROBJ11"
2442 }
2443 },
2444 {
2445 "shards": [
2446 {
2447 "errors": [
2448 "stat_error"
2449 ],
2450 "osd": 0,
2451 "primary": false
2452 },
2453 {
2454 "data_digest": "0x2ddbf8f5",
2455 "omap_digest": "0x067f306a",
2456 "size": 7,
2457 "errors": [],
2458 "osd": 1,
2459 "primary": true
2460 }
2461 ],
2462 "selected_object_info": {
2463 "oid": {
2464 "oid": "ROBJ12",
2465 "key": "",
2466 "snapid": -2,
2467 "hash": 3920199997,
2468 "max": 0,
2469 "pool": 3,
2470 "namespace": ""
2471 },
2472 "version": "51'56",
2473 "prior_version": "43'36",
2474 "last_reqid": "osd.1.0:55",
2475 "user_version": 36,
2476 "size": 7,
2477 "mtime": "2018-04-05 14:33:27.460958",
2478 "local_mtime": "2018-04-05 14:33:27.462109",
2479 "lost": 0,
2480 "flags": [
2481 "dirty",
2482 "omap",
2483 "data_digest",
2484 "omap_digest"
2485 ],
2486 "truncate_seq": 0,
2487 "truncate_size": 0,
2488 "data_digest": "0x2ddbf8f5",
2489 "omap_digest": "0x067f306a",
2490 "expected_object_size": 0,
2491 "expected_write_size": 0,
2492 "alloc_hint_flags": 0,
2493 "manifest": {
2494 "type": 0
2495 },
2496 "watchers": {}
2497 },
2498 "union_shard_errors": [
2499 "stat_error"
2500 ],
2501 "errors": [],
2502 "object": {
2503 "version": 36,
2504 "snap": "head",
2505 "locator": "",
2506 "nspace": "",
2507 "name": "ROBJ12"
2508 }
2509 },
2510 {
2511 "shards": [
2512 {
2513 "errors": [
2514 "stat_error"
2515 ],
2516 "osd": 0,
2517 "primary": false
2518 },
2519 {
2520 "size": 7,
2521 "errors": [
2522 "read_error"
2523 ],
2524 "osd": 1,
2525 "primary": true
2526 }
2527 ],
2528 "union_shard_errors": [
2529 "stat_error",
2530 "read_error"
2531 ],
2532 "errors": [],
2533 "object": {
2534 "version": 0,
2535 "snap": "head",
2536 "locator": "",
2537 "nspace": "",
2538 "name": "ROBJ13"
2539 }
2540 },
2541 {
2542 "shards": [
2543 {
2544 "object_info": "bad-val",
2545 "data_digest": "0x2ddbf8f5",
2546 "omap_digest": "0x4f14f849",
2547 "size": 7,
2548 "errors": [
2549 "info_corrupted"
2550 ],
2551 "osd": 0,
2552 "primary": false
2553 },
2554 {
2555 "data_digest": "0x2ddbf8f5",
2556 "omap_digest": "0x4f14f849",
2557 "size": 7,
2558 "errors": [
2559 "info_missing"
2560 ],
2561 "osd": 1,
2562 "primary": true
2563 }
2564 ],
2565 "union_shard_errors": [
2566 "info_missing",
2567 "info_corrupted"
2568 ],
2569 "errors": [],
2570 "object": {
2571 "version": 0,
2572 "snap": "head",
2573 "locator": "",
2574 "nspace": "",
2575 "name": "ROBJ14"
2576 }
2577 },
2578 {
2579 "shards": [
2580 {
2581 "object_info": {
2582 "oid": {
2583 "oid": "ROBJ15",
2584 "key": "",
2585 "snapid": -2,
2586 "hash": 504996876,
2587 "max": 0,
2588 "pool": 3,
2589 "namespace": ""
2590 },
2591 "version": "51'49",
2592 "prior_version": "49'45",
2593 "last_reqid": "osd.1.0:48",
2594 "user_version": 45,
2595 "size": 7,
2596 "mtime": "2018-04-05 14:33:29.498969",
2597 "local_mtime": "2018-04-05 14:33:29.499890",
2598 "lost": 0,
2599 "flags": [
2600 "dirty",
2601 "omap",
2602 "data_digest",
2603 "omap_digest"
2604 ],
2605 "truncate_seq": 0,
2606 "truncate_size": 0,
2607 "data_digest": "0x2ddbf8f5",
2608 "omap_digest": "0x2d2a4d6e",
2609 "expected_object_size": 0,
2610 "expected_write_size": 0,
2611 "alloc_hint_flags": 0,
2612 "manifest": {
2613 "type": 0
2614 },
2615 "watchers": {}
2616 },
2617 "data_digest": "0x2ddbf8f5",
2618 "omap_digest": "0x2d2a4d6e",
2619 "size": 7,
2620 "errors": [],
2621 "osd": 0,
2622 "primary": false
2623 },
2624 {
2625 "data_digest": "0x2ddbf8f5",
2626 "omap_digest": "0x2d2a4d6e",
2627 "size": 7,
2628 "errors": [
2629 "info_missing"
2630 ],
2631 "osd": 1,
2632 "primary": true
2633 }
2634 ],
2635 "selected_object_info": {
2636 "oid": {
2637 "oid": "ROBJ15",
2638 "key": "",
2639 "snapid": -2,
2640 "hash": 504996876,
2641 "max": 0,
2642 "pool": 3,
2643 "namespace": ""
2644 },
2645 "version": "51'49",
2646 "prior_version": "49'45",
2647 "last_reqid": "osd.1.0:48",
2648 "user_version": 45,
2649 "size": 7,
2650 "mtime": "2018-04-05 14:33:29.498969",
2651 "local_mtime": "2018-04-05 14:33:29.499890",
2652 "lost": 0,
2653 "flags": [
2654 "dirty",
2655 "omap",
2656 "data_digest",
2657 "omap_digest"
2658 ],
2659 "truncate_seq": 0,
2660 "truncate_size": 0,
2661 "data_digest": "0x2ddbf8f5",
2662 "omap_digest": "0x2d2a4d6e",
2663 "expected_object_size": 0,
2664 "expected_write_size": 0,
2665 "alloc_hint_flags": 0,
2666 "manifest": {
2667 "type": 0
2668 },
2669 "watchers": {}
2670 },
2671 "union_shard_errors": [
2672 "info_missing"
2673 ],
2674 "errors": [],
2675 "object": {
2676 "version": 45,
2677 "snap": "head",
2678 "locator": "",
2679 "nspace": "",
2680 "name": "ROBJ15"
2681 }
2682 },
2683 {
2684 "errors": [],
2685 "object": {
2686 "locator": "",
2687 "name": "ROBJ16",
2688 "nspace": "",
2689 "snap": "head",
2690 "version": 0
2691 },
2692 "shards": [
2693 {
2694 "data_digest": "0x2ddbf8f5",
2695 "errors": [
2696 "snapset_missing"
2697 ],
2698 "omap_digest": "0x8b699207",
2699 "osd": 0,
2700 "primary": false,
2701 "size": 7
2702 },
2703 {
2704 "snapset": "bad-val",
2705 "data_digest": "0x2ddbf8f5",
2706 "errors": [
2707 "snapset_corrupted"
2708 ],
2709 "omap_digest": "0x8b699207",
2710 "osd": 1,
2711 "primary": true,
2712 "size": 7
2713 }
2714 ],
2715 "union_shard_errors": [
2716 "snapset_missing",
2717 "snapset_corrupted"
2718 ]
2719 },
2720 {
2721 "errors": [],
2722 "object": {
2723 "locator": "",
2724 "name": "ROBJ17",
2725 "nspace": "",
2726 "snap": "head"
2727 },
2728 "selected_object_info": {
2729 "alloc_hint_flags": 0,
2730 "data_digest": "0x2ddbf8f5",
2731 "expected_object_size": 0,
2732 "expected_write_size": 0,
2733 "flags": [
2734 "dirty",
2735 "omap",
2736 "data_digest",
2737 "omap_digest"
2738 ],
2739 "lost": 0,
2740 "manifest": {
2741 "type": 0
2742 },
2743 "oid": {
2744 "hash": 1884071249,
2745 "key": "",
2746 "max": 0,
2747 "namespace": "",
2748 "oid": "ROBJ17",
2749 "pool": 3,
2750 "snapid": -2
2751 },
2752 "omap_digest": "0xe9572720",
2753 "size": 7,
2754 "truncate_seq": 0,
2755 "truncate_size": 0,
2756 "user_version": 51,
2757 "watchers": {}
2758 },
2759 "shards": [
2760 {
2761 "data_digest": "0x5af0c3ef",
2762 "errors": [
2763 "data_digest_mismatch_info"
2764 ],
2765 "omap_digest": "0xe9572720",
2766 "osd": 0,
2767 "primary": false,
2768 "size": 7
2769 },
2770 {
2771 "data_digest": "0x5af0c3ef",
2772 "errors": [
2773 "data_digest_mismatch_info"
2774 ],
2775 "omap_digest": "0xe9572720",
2776 "osd": 1,
2777 "primary": true,
2778 "size": 7
2779 }
2780 ],
2781 "union_shard_errors": [
2782 "data_digest_mismatch_info"
2783 ]
2784 },
2785 {
2786 "errors": [
2787 "object_info_inconsistency"
2788 ],
2789 "object": {
2790 "locator": "",
2791 "name": "ROBJ18",
2792 "nspace": "",
2793 "snap": "head"
2794 },
2795 "selected_object_info": {
2796 "alloc_hint_flags": 255,
2797 "data_digest": "0x2ddbf8f5",
2798 "expected_object_size": 0,
2799 "expected_write_size": 0,
2800 "flags": [
2801 "dirty",
2802 "omap",
2803 "data_digest",
2804 "omap_digest"
2805 ],
2806 "lost": 0,
2807 "manifest": {
2808 "type": 0
2809 },
2810 "oid": {
2811 "hash": 1629828556,
2812 "key": "",
2813 "max": 0,
2814 "namespace": "",
2815 "oid": "ROBJ18",
2816 "pool": 3,
2817 "snapid": -2
2818 },
2819 "omap_digest": "0xddc3680f",
2820 "size": 7,
2821 "truncate_seq": 0,
2822 "truncate_size": 0,
2823 "user_version": 54,
2824 "watchers": {}
2825 },
2826 "shards": [
2827 {
2828 "data_digest": "0xbd89c912",
2829 "errors": [
2830 "data_digest_mismatch_info"
2831 ],
2832 "object_info": {
2833 "alloc_hint_flags": 0,
2834 "data_digest": "0x2ddbf8f5",
2835 "expected_object_size": 0,
2836 "expected_write_size": 0,
2837 "flags": [
2838 "dirty",
2839 "omap",
2840 "data_digest",
2841 "omap_digest"
2842 ],
2843 "lost": 0,
2844 "manifest": {
2845 "type": 0
2846 },
2847 "oid": {
2848 "hash": 1629828556,
2849 "key": "",
2850 "max": 0,
2851 "namespace": "",
2852 "oid": "ROBJ18",
2853 "pool": 3,
2854 "snapid": -2
2855 },
2856 "omap_digest": "0xddc3680f",
2857 "size": 7,
2858 "truncate_seq": 0,
2859 "truncate_size": 0,
2860 "user_version": 54,
2861 "watchers": {}
2862 },
2863 "omap_digest": "0xddc3680f",
2864 "osd": 0,
2865 "primary": false,
2866 "size": 7
2867 },
2868 {
2869 "data_digest": "0xbd89c912",
2870 "errors": [
2871 "data_digest_mismatch_info"
2872 ],
2873 "object_info": {
2874 "alloc_hint_flags": 255,
2875 "data_digest": "0x2ddbf8f5",
2876 "expected_object_size": 0,
2877 "expected_write_size": 0,
2878 "flags": [
2879 "dirty",
2880 "omap",
2881 "data_digest",
2882 "omap_digest"
2883 ],
2884 "lost": 0,
2885 "manifest": {
2886 "type": 0
2887 },
2888 "oid": {
2889 "hash": 1629828556,
2890 "key": "",
2891 "max": 0,
2892 "namespace": "",
2893 "oid": "ROBJ18",
2894 "pool": 3,
2895 "snapid": -2
2896 },
2897 "omap_digest": "0xddc3680f",
2898 "size": 7,
2899 "truncate_seq": 0,
2900 "truncate_size": 0,
2901 "user_version": 54,
2902 "watchers": {}
2903 },
2904 "omap_digest": "0xddc3680f",
2905 "osd": 1,
2906 "primary": true,
2907 "size": 7
2908 }
2909 ],
2910 "union_shard_errors": [
2911 "data_digest_mismatch_info"
2912 ]
2913 },
2914 {
2915 "shards": [
2916 {
2917 "data_digest": "0x578a4830",
2918 "omap_digest": "0xf8e11918",
2919 "size": 7,
2920 "errors": [
2921 "data_digest_mismatch_info"
2922 ],
2923 "osd": 0,
2924 "primary": false
2925 },
2926 {
2927 "data_digest": "0x2ddbf8f5",
2928 "omap_digest": "0xf8e11918",
2929 "size": 7,
2930 "errors": [],
2931 "osd": 1,
2932 "primary": true
2933 }
2934 ],
2935 "selected_object_info": {
2936 "oid": {
2937 "oid": "ROBJ2",
2938 "key": "",
2939 "snapid": -2,
2940 "hash": 2026323607,
2941 "max": 0,
2942 "pool": 3,
2943 "namespace": ""
2944 },
2945 "version": "51'60",
2946 "prior_version": "23'6",
2947 "last_reqid": "osd.1.0:59",
2948 "user_version": 6,
2949 "size": 7,
2950 "mtime": "2018-04-05 14:33:20.498756",
2951 "local_mtime": "2018-04-05 14:33:20.499704",
2952 "lost": 0,
2953 "flags": [
2954 "dirty",
2955 "omap",
2956 "data_digest",
2957 "omap_digest"
2958 ],
2959 "truncate_seq": 0,
2960 "truncate_size": 0,
2961 "data_digest": "0x2ddbf8f5",
2962 "omap_digest": "0xf8e11918",
2963 "expected_object_size": 0,
2964 "expected_write_size": 0,
2965 "alloc_hint_flags": 0,
2966 "manifest": {
2967 "type": 0
2968 },
2969 "watchers": {}
2970 },
2971 "union_shard_errors": [
2972 "data_digest_mismatch_info"
2973 ],
2974 "errors": [
2975 "data_digest_mismatch"
2976 ],
2977 "object": {
2978 "version": 6,
2979 "snap": "head",
2980 "locator": "",
2981 "nspace": "",
2982 "name": "ROBJ2"
2983 }
2984 },
2985 {
2986 "shards": [
2987 {
2988 "data_digest": "0x2ddbf8f5",
2989 "omap_digest": "0x00b35dfd",
2990 "size": 7,
2991 "errors": [],
2992 "osd": 0,
2993 "primary": false
2994 },
2995 {
2996 "errors": [
2997 "missing"
2998 ],
2999 "osd": 1,
3000 "primary": true
3001 }
3002 ],
3003 "selected_object_info": {
3004 "oid": {
3005 "oid": "ROBJ3",
3006 "key": "",
3007 "snapid": -2,
3008 "hash": 625845583,
3009 "max": 0,
3010 "pool": 3,
3011 "namespace": ""
3012 },
3013 "version": "51'61",
3014 "prior_version": "25'9",
3015 "last_reqid": "osd.1.0:60",
3016 "user_version": 9,
3017 "size": 7,
3018 "mtime": "2018-04-05 14:33:21.189382",
3019 "local_mtime": "2018-04-05 14:33:21.190446",
3020 "lost": 0,
3021 "flags": [
3022 "dirty",
3023 "omap",
3024 "data_digest",
3025 "omap_digest"
3026 ],
3027 "truncate_seq": 0,
3028 "truncate_size": 0,
3029 "data_digest": "0x2ddbf8f5",
3030 "omap_digest": "0x00b35dfd",
3031 "expected_object_size": 0,
3032 "expected_write_size": 0,
3033 "alloc_hint_flags": 0,
3034 "manifest": {
3035 "type": 0
3036 },
3037 "watchers": {}
3038 },
3039 "union_shard_errors": [
3040 "missing"
3041 ],
3042 "errors": [],
3043 "object": {
3044 "version": 9,
3045 "snap": "head",
3046 "locator": "",
3047 "nspace": "",
3048 "name": "ROBJ3"
3049 }
3050 },
3051 {
3052 "shards": [
3053 {
3054 "data_digest": "0x2ddbf8f5",
3055 "omap_digest": "0xd7178dfe",
3056 "size": 7,
3057 "errors": [
3058 "omap_digest_mismatch_info"
3059 ],
3060 "osd": 0,
3061 "primary": false
3062 },
3063 {
3064 "data_digest": "0x2ddbf8f5",
3065 "omap_digest": "0xe2d46ea4",
3066 "size": 7,
3067 "errors": [],
3068 "osd": 1,
3069 "primary": true
3070 }
3071 ],
3072 "selected_object_info": {
3073 "oid": {
3074 "oid": "ROBJ4",
3075 "key": "",
3076 "snapid": -2,
3077 "hash": 2360875311,
3078 "max": 0,
3079 "pool": 3,
3080 "namespace": ""
3081 },
3082 "version": "51'62",
3083 "prior_version": "27'12",
3084 "last_reqid": "osd.1.0:61",
3085 "user_version": 12,
3086 "size": 7,
3087 "mtime": "2018-04-05 14:33:21.862313",
3088 "local_mtime": "2018-04-05 14:33:21.863261",
3089 "lost": 0,
3090 "flags": [
3091 "dirty",
3092 "omap",
3093 "data_digest",
3094 "omap_digest"
3095 ],
3096 "truncate_seq": 0,
3097 "truncate_size": 0,
3098 "data_digest": "0x2ddbf8f5",
3099 "omap_digest": "0xe2d46ea4",
3100 "expected_object_size": 0,
3101 "expected_write_size": 0,
3102 "alloc_hint_flags": 0,
3103 "manifest": {
3104 "type": 0
3105 },
3106 "watchers": {}
3107 },
3108 "union_shard_errors": [
3109 "omap_digest_mismatch_info"
3110 ],
3111 "errors": [
3112 "omap_digest_mismatch"
3113 ],
3114 "object": {
3115 "version": 12,
3116 "snap": "head",
3117 "locator": "",
3118 "nspace": "",
3119 "name": "ROBJ4"
3120 }
3121 },
3122 {
3123 "shards": [
3124 {
3125 "data_digest": "0x2ddbf8f5",
3126 "omap_digest": "0x1a862a41",
3127 "size": 7,
3128 "errors": [],
3129 "osd": 0,
3130 "primary": false
3131 },
3132 {
3133 "data_digest": "0x2ddbf8f5",
3134 "omap_digest": "0x06cac8f6",
3135 "size": 7,
3136 "errors": [
3137 "omap_digest_mismatch_info"
3138 ],
3139 "osd": 1,
3140 "primary": true
3141 }
3142 ],
3143 "selected_object_info": {
3144 "oid": {
3145 "oid": "ROBJ5",
3146 "key": "",
3147 "snapid": -2,
3148 "hash": 2334915887,
3149 "max": 0,
3150 "pool": 3,
3151 "namespace": ""
3152 },
3153 "version": "51'63",
3154 "prior_version": "29'15",
3155 "last_reqid": "osd.1.0:62",
3156 "user_version": 15,
3157 "size": 7,
3158 "mtime": "2018-04-05 14:33:22.589300",
3159 "local_mtime": "2018-04-05 14:33:22.590376",
3160 "lost": 0,
3161 "flags": [
3162 "dirty",
3163 "omap",
3164 "data_digest",
3165 "omap_digest"
3166 ],
3167 "truncate_seq": 0,
3168 "truncate_size": 0,
3169 "data_digest": "0x2ddbf8f5",
3170 "omap_digest": "0x1a862a41",
3171 "expected_object_size": 0,
3172 "expected_write_size": 0,
3173 "alloc_hint_flags": 0,
3174 "manifest": {
3175 "type": 0
3176 },
3177 "watchers": {}
3178 },
3179 "union_shard_errors": [
3180 "omap_digest_mismatch_info"
3181 ],
3182 "errors": [
3183 "omap_digest_mismatch"
3184 ],
3185 "object": {
3186 "version": 15,
3187 "snap": "head",
3188 "locator": "",
3189 "nspace": "",
3190 "name": "ROBJ5"
3191 }
3192 },
3193 {
3194 "shards": [
3195 {
3196 "data_digest": "0x2ddbf8f5",
3197 "omap_digest": "0x689ee887",
3198 "size": 7,
3199 "errors": [
3200 "omap_digest_mismatch_info"
3201 ],
3202 "osd": 0,
3203 "primary": false
3204 },
3205 {
3206 "data_digest": "0x2ddbf8f5",
3207 "omap_digest": "0x179c919f",
3208 "size": 7,
3209 "errors": [],
3210 "osd": 1,
3211 "primary": true
3212 }
3213 ],
3214 "selected_object_info": {
3215 "oid": {
3216 "oid": "ROBJ6",
3217 "key": "",
3218 "snapid": -2,
3219 "hash": 390610085,
3220 "max": 0,
3221 "pool": 3,
3222 "namespace": ""
3223 },
3224 "version": "51'54",
3225 "prior_version": "31'18",
3226 "last_reqid": "osd.1.0:53",
3227 "user_version": 18,
3228 "size": 7,
3229 "mtime": "2018-04-05 14:33:23.289188",
3230 "local_mtime": "2018-04-05 14:33:23.290130",
3231 "lost": 0,
3232 "flags": [
3233 "dirty",
3234 "omap",
3235 "data_digest",
3236 "omap_digest"
3237 ],
3238 "truncate_seq": 0,
3239 "truncate_size": 0,
3240 "data_digest": "0x2ddbf8f5",
3241 "omap_digest": "0x179c919f",
3242 "expected_object_size": 0,
3243 "expected_write_size": 0,
3244 "alloc_hint_flags": 0,
3245 "manifest": {
3246 "type": 0
3247 },
3248 "watchers": {}
3249 },
3250 "union_shard_errors": [
3251 "omap_digest_mismatch_info"
3252 ],
3253 "errors": [
3254 "omap_digest_mismatch"
3255 ],
3256 "object": {
3257 "version": 18,
3258 "snap": "head",
3259 "locator": "",
3260 "nspace": "",
3261 "name": "ROBJ6"
3262 }
3263 },
3264 {
3265 "shards": [
3266 {
3267 "data_digest": "0x2ddbf8f5",
3268 "omap_digest": "0xefced57a",
3269 "size": 7,
3270 "errors": [],
3271 "osd": 0,
3272 "primary": false
3273 },
3274 {
3275 "data_digest": "0x2ddbf8f5",
3276 "omap_digest": "0x6a73cc07",
3277 "size": 7,
3278 "errors": [
3279 "omap_digest_mismatch_info"
3280 ],
3281 "osd": 1,
3282 "primary": true
3283 }
3284 ],
3285 "selected_object_info": {
3286 "oid": {
3287 "oid": "ROBJ7",
3288 "key": "",
3289 "snapid": -2,
3290 "hash": 3529485009,
3291 "max": 0,
3292 "pool": 3,
3293 "namespace": ""
3294 },
3295 "version": "51'53",
3296 "prior_version": "33'21",
3297 "last_reqid": "osd.1.0:52",
3298 "user_version": 21,
3299 "size": 7,
3300 "mtime": "2018-04-05 14:33:23.979658",
3301 "local_mtime": "2018-04-05 14:33:23.980731",
3302 "lost": 0,
3303 "flags": [
3304 "dirty",
3305 "omap",
3306 "data_digest",
3307 "omap_digest"
3308 ],
3309 "truncate_seq": 0,
3310 "truncate_size": 0,
3311 "data_digest": "0x2ddbf8f5",
3312 "omap_digest": "0xefced57a",
3313 "expected_object_size": 0,
3314 "expected_write_size": 0,
3315 "alloc_hint_flags": 0,
3316 "manifest": {
3317 "type": 0
3318 },
3319 "watchers": {}
3320 },
3321 "union_shard_errors": [
3322 "omap_digest_mismatch_info"
3323 ],
3324 "errors": [
3325 "omap_digest_mismatch"
3326 ],
3327 "object": {
3328 "version": 21,
3329 "snap": "head",
3330 "locator": "",
3331 "nspace": "",
3332 "name": "ROBJ7"
3333 }
3334 },
3335 {
3336 "shards": [
3337 {
3338 "attrs": [
3339 {
3340 "Base64": false,
3341 "value": "bad-val",
3342 "name": "key1-ROBJ8"
3343 },
3344 {
3345 "Base64": false,
3346 "value": "val2-ROBJ8",
3347 "name": "key2-ROBJ8"
3348 }
3349 ],
3350 "data_digest": "0x2ddbf8f5",
3351 "omap_digest": "0xd6be81dc",
3352 "size": 7,
3353 "errors": [],
3354 "osd": 0,
3355 "primary": false
3356 },
3357 {
3358 "attrs": [
3359 {
3360 "Base64": false,
3361 "value": "val1-ROBJ8",
3362 "name": "key1-ROBJ8"
3363 },
3364 {
3365 "Base64": false,
3366 "value": "val3-ROBJ8",
3367 "name": "key3-ROBJ8"
3368 }
3369 ],
3370 "data_digest": "0x2ddbf8f5",
3371 "omap_digest": "0xd6be81dc",
3372 "size": 7,
3373 "errors": [],
3374 "osd": 1,
3375 "primary": true
3376 }
3377 ],
3378 "selected_object_info": {
3379 "oid": {
3380 "oid": "ROBJ8",
3381 "key": "",
3382 "snapid": -2,
3383 "hash": 2359695969,
3384 "max": 0,
3385 "pool": 3,
3386 "namespace": ""
3387 },
3388 "version": "79'66",
3389 "prior_version": "79'65",
3390 "last_reqid": "client.4554.0:1",
3391 "user_version": 79,
3392 "size": 7,
3393 "mtime": "2018-04-05 14:34:05.598688",
3394 "local_mtime": "2018-04-05 14:34:05.599698",
3395 "lost": 0,
3396 "flags": [
3397 "dirty",
3398 "omap",
3399 "data_digest",
3400 "omap_digest"
3401 ],
3402 "truncate_seq": 0,
3403 "truncate_size": 0,
3404 "data_digest": "0x2ddbf8f5",
3405 "omap_digest": "0xd6be81dc",
3406 "expected_object_size": 0,
3407 "expected_write_size": 0,
3408 "alloc_hint_flags": 0,
3409 "manifest": {
3410 "type": 0
3411 },
3412 "watchers": {}
3413 },
3414 "union_shard_errors": [],
3415 "errors": [
3416 "attr_value_mismatch",
3417 "attr_name_mismatch"
3418 ],
3419 "object": {
3420 "version": 66,
3421 "snap": "head",
3422 "locator": "",
3423 "nspace": "",
3424 "name": "ROBJ8"
3425 }
3426 },
3427 {
3428 "shards": [
3429 {
3430 "object_info": {
3431 "oid": {
3432 "oid": "ROBJ9",
3433 "key": "",
3434 "snapid": -2,
3435 "hash": 537189375,
3436 "max": 0,
3437 "pool": 3,
3438 "namespace": ""
3439 },
3440 "version": "51'64",
3441 "prior_version": "37'27",
3442 "last_reqid": "osd.1.0:63",
3443 "user_version": 27,
3444 "size": 7,
3445 "mtime": "2018-04-05 14:33:25.352485",
3446 "local_mtime": "2018-04-05 14:33:25.353746",
3447 "lost": 0,
3448 "flags": [
3449 "dirty",
3450 "omap",
3451 "data_digest",
3452 "omap_digest"
3453 ],
3454 "truncate_seq": 0,
3455 "truncate_size": 0,
3456 "data_digest": "0x2ddbf8f5",
3457 "omap_digest": "0x2eecc539",
3458 "expected_object_size": 0,
3459 "expected_write_size": 0,
3460 "alloc_hint_flags": 0,
3461 "manifest": {
3462 "type": 0
3463 },
3464 "watchers": {}
3465 },
3466 "data_digest": "0x1f26fb26",
3467 "omap_digest": "0x2eecc539",
3468 "size": 3,
3469 "errors": [
3470 "obj_size_info_mismatch"
3471 ],
3472 "osd": 0,
3473 "primary": false
3474 },
3475 {
3476 "object_info": {
3477 "oid": {
3478 "oid": "ROBJ9",
3479 "key": "",
3480 "snapid": -2,
3481 "hash": 537189375,
3482 "max": 0,
3483 "pool": 3,
3484 "namespace": ""
3485 },
3486 "version": "119'68",
3487 "prior_version": "51'64",
3488 "last_reqid": "client.4834.0:1",
3489 "user_version": 81,
3490 "size": 3,
3491 "mtime": "2018-04-05 14:35:01.500659",
3492 "local_mtime": "2018-04-05 14:35:01.502117",
3493 "lost": 0,
3494 "flags": [
3495 "dirty",
3496 "omap",
3497 "data_digest",
3498 "omap_digest"
3499 ],
3500 "truncate_seq": 0,
3501 "truncate_size": 0,
3502 "data_digest": "0x1f26fb26",
3503 "omap_digest": "0x2eecc539",
3504 "expected_object_size": 0,
3505 "expected_write_size": 0,
3506 "alloc_hint_flags": 0,
3507 "manifest": {
3508 "type": 0
3509 },
3510 "watchers": {}
3511 },
3512 "data_digest": "0x1f26fb26",
3513 "omap_digest": "0x2eecc539",
3514 "size": 3,
3515 "errors": [],
3516 "osd": 1,
3517 "primary": true
3518 }
3519 ],
3520 "selected_object_info": {
3521 "oid": {
3522 "oid": "ROBJ9",
3523 "key": "",
3524 "snapid": -2,
3525 "hash": 537189375,
3526 "max": 0,
3527 "pool": 3,
3528 "namespace": ""
3529 },
3530 "version": "119'68",
3531 "prior_version": "51'64",
3532 "last_reqid": "client.4834.0:1",
3533 "user_version": 81,
3534 "size": 3,
3535 "mtime": "2018-04-05 14:35:01.500659",
3536 "local_mtime": "2018-04-05 14:35:01.502117",
3537 "lost": 0,
3538 "flags": [
3539 "dirty",
3540 "omap",
3541 "data_digest",
3542 "omap_digest"
3543 ],
3544 "truncate_seq": 0,
3545 "truncate_size": 0,
3546 "data_digest": "0x1f26fb26",
3547 "omap_digest": "0x2eecc539",
3548 "expected_object_size": 0,
3549 "expected_write_size": 0,
3550 "alloc_hint_flags": 0,
3551 "manifest": {
3552 "type": 0
3553 },
3554 "watchers": {}
3555 },
3556 "union_shard_errors": [
3557 "obj_size_info_mismatch"
3558 ],
3559 "errors": [
3560 "object_info_inconsistency"
3561 ],
3562 "object": {
3563 "version": 68,
3564 "snap": "head",
3565 "locator": "",
3566 "nspace": "",
3567 "name": "ROBJ9"
3568 }
3569 }
3570 ],
3571 "epoch": 0
3572 }
3573 EOF
3574
3575 jq "$jqfilter" $dir/json | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/csjson
3576 multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
3577 if test $getjson = "yes"
3578 then
3579 jq '.' $dir/json > save2.json
3580 fi
3581
3582 if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
3583 then
3584 jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
3585 fi
3586
3587 repair $pg
3588 wait_for_clean
3589
3590 # This hangs if the repair doesn't work
3591 timeout 30 rados -p $poolname get ROBJ17 $dir/robj17.out || return 1
3592 timeout 30 rados -p $poolname get ROBJ18 $dir/robj18.out || return 1
3593 # Even though we couldn't repair all of the introduced errors, we can fix ROBJ17
3594 diff -q $dir/new.ROBJ17 $dir/robj17.out || return 1
3595 rm -f $dir/new.ROBJ17 $dir/robj17.out || return 1
3596 diff -q $dir/new.ROBJ18 $dir/robj18.out || return 1
3597 rm -f $dir/new.ROBJ18 $dir/robj18.out || return 1
3598
3599 if [ $ERRORS != "0" ];
3600 then
3601 echo "TEST FAILED WITH $ERRORS ERRORS"
3602 return 1
3603 fi
3604
3605 ceph osd pool rm $poolname $poolname --yes-i-really-really-mean-it
3606 }
3607
3608
3609 #
3610 # Test scrub errors for an erasure coded pool
3611 #
3612 function corrupt_scrub_erasure() {
3613 local dir=$1
3614 local allow_overwrites=$2
3615 local poolname=ecpool
3616 local total_objs=7
3617
3618 run_mon $dir a || return 1
3619 run_mgr $dir x || return 1
3620 for id in $(seq 0 2) ; do
3621 if [ "$allow_overwrites" = "true" ]; then
3622 run_osd $dir $id || return 1
3623 else
3624 run_osd_filestore $dir $id || return 1
3625 fi
3626 done
3627 create_rbd_pool || return 1
3628 create_pool foo 1
3629
3630 create_ec_pool $poolname $allow_overwrites k=2 m=1 stripe_unit=2K --force || return 1
3631 wait_for_clean || return 1
3632
3633 for i in $(seq 1 $total_objs) ; do
3634 objname=EOBJ${i}
3635 add_something $dir $poolname $objname || return 1
3636
3637 local osd=$(expr $i % 2)
3638
3639 case $i in
3640 1)
3641 # Size (deep scrub data_digest too)
3642 local payload=UVWXYZZZ
3643 echo $payload > $dir/CORRUPT
3644 objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
3645 ;;
3646
3647 2)
3648 # Corrupt EC shard
3649 dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=1
3650 objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
3651 ;;
3652
3653 3)
3654 # missing
3655 objectstore_tool $dir $osd $objname remove || return 1
3656 ;;
3657
3658 4)
3659 rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1
3660 rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1
3661
3662 # Break xattrs
3663 echo -n bad-val > $dir/bad-val
3664 objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1
3665 objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1
3666 echo -n val3-$objname > $dir/newval
3667 objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1
3668 rm $dir/bad-val $dir/newval
3669 ;;
3670
3671 5)
3672 # Corrupt EC shard
3673 dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=2
3674 objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
3675 ;;
3676
3677 6)
3678 objectstore_tool $dir 0 $objname rm-attr hinfo_key || return 1
3679 echo -n bad-val > $dir/bad-val
3680 objectstore_tool $dir 1 $objname set-attr hinfo_key $dir/bad-val || return 1
3681 ;;
3682
3683 7)
3684 local payload=MAKETHISDIFFERENTFROMOTHEROBJECTS
3685 echo $payload > $dir/DIFFERENT
3686 rados --pool $poolname put $objname $dir/DIFFERENT || return 1
3687
3688 # Get hinfo_key from EOBJ1
3689 objectstore_tool $dir 0 EOBJ1 get-attr hinfo_key > $dir/hinfo
3690 objectstore_tool $dir 0 $objname set-attr hinfo_key $dir/hinfo || return 1
3691 rm -f $dir/hinfo
3692 ;;
3693
3694 esac
3695 done
3696
3697 local pg=$(get_pg $poolname EOBJ0)
3698
3699 pg_scrub $pg
3700
3701 rados list-inconsistent-pg $poolname > $dir/json || return 1
3702 # Check pg count
3703 test $(jq '. | length' $dir/json) = "1" || return 1
3704 # Check pgid
3705 test $(jq -r '.[0]' $dir/json) = $pg || return 1
3706
3707 rados list-inconsistent-obj $pg > $dir/json || return 1
3708 # Get epoch for repair-get requests
3709 epoch=$(jq .epoch $dir/json)
3710
3711 jq "$jqfilter" << EOF | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/checkcsjson
3712 {
3713 "inconsistents": [
3714 {
3715 "shards": [
3716 {
3717 "size": 2048,
3718 "errors": [],
3719 "shard": 2,
3720 "osd": 0,
3721 "primary": false
3722 },
3723 {
3724 "object_info": {
3725 "oid": {
3726 "oid": "EOBJ1",
3727 "key": "",
3728 "snapid": -2,
3729 "hash": 560836233,
3730 "max": 0,
3731 "pool": 3,
3732 "namespace": ""
3733 },
3734 "version": "27'1",
3735 "prior_version": "0'0",
3736 "last_reqid": "client.4184.0:1",
3737 "user_version": 1,
3738 "size": 7,
3739 "mtime": "",
3740 "local_mtime": "",
3741 "lost": 0,
3742 "flags": [
3743 "dirty",
3744 "data_digest"
3745 ],
3746 "truncate_seq": 0,
3747 "truncate_size": 0,
3748 "data_digest": "0x2ddbf8f5",
3749 "omap_digest": "0xffffffff",
3750 "expected_object_size": 0,
3751 "expected_write_size": 0,
3752 "alloc_hint_flags": 0,
3753 "manifest": {
3754 "type": 0
3755 },
3756 "watchers": {}
3757 },
3758 "size": 9,
3759 "shard": 0,
3760 "errors": [
3761 "size_mismatch_info",
3762 "obj_size_info_mismatch"
3763 ],
3764 "osd": 1,
3765 "primary": true
3766 },
3767 {
3768 "size": 2048,
3769 "shard": 1,
3770 "errors": [],
3771 "osd": 2,
3772 "primary": false
3773 }
3774 ],
3775 "selected_object_info": {
3776 "oid": {
3777 "oid": "EOBJ1",
3778 "key": "",
3779 "snapid": -2,
3780 "hash": 560836233,
3781 "max": 0,
3782 "pool": 3,
3783 "namespace": ""
3784 },
3785 "version": "27'1",
3786 "prior_version": "0'0",
3787 "last_reqid": "client.4184.0:1",
3788 "user_version": 1,
3789 "size": 7,
3790 "mtime": "",
3791 "local_mtime": "",
3792 "lost": 0,
3793 "flags": [
3794 "dirty",
3795 "data_digest"
3796 ],
3797 "truncate_seq": 0,
3798 "truncate_size": 0,
3799 "data_digest": "0x2ddbf8f5",
3800 "omap_digest": "0xffffffff",
3801 "expected_object_size": 0,
3802 "expected_write_size": 0,
3803 "alloc_hint_flags": 0,
3804 "manifest": {
3805 "type": 0
3806 },
3807 "watchers": {}
3808 },
3809 "union_shard_errors": [
3810 "size_mismatch_info",
3811 "obj_size_info_mismatch"
3812 ],
3813 "errors": [
3814 "size_mismatch"
3815 ],
3816 "object": {
3817 "version": 1,
3818 "snap": "head",
3819 "locator": "",
3820 "nspace": "",
3821 "name": "EOBJ1"
3822 }
3823 },
3824 {
3825 "shards": [
3826 {
3827 "size": 2048,
3828 "errors": [],
3829 "shard": 2,
3830 "osd": 0,
3831 "primary": false
3832 },
3833 {
3834 "shard": 0,
3835 "errors": [
3836 "missing"
3837 ],
3838 "osd": 1,
3839 "primary": true
3840 },
3841 {
3842 "size": 2048,
3843 "shard": 1,
3844 "errors": [],
3845 "osd": 2,
3846 "primary": false
3847 }
3848 ],
3849 "selected_object_info": {
3850 "oid": {
3851 "oid": "EOBJ3",
3852 "key": "",
3853 "snapid": -2,
3854 "hash": 3125668237,
3855 "max": 0,
3856 "pool": 3,
3857 "namespace": ""
3858 },
3859 "version": "39'3",
3860 "prior_version": "0'0",
3861 "last_reqid": "client.4252.0:1",
3862 "user_version": 3,
3863 "size": 7,
3864 "mtime": "",
3865 "local_mtime": "",
3866 "lost": 0,
3867 "flags": [
3868 "dirty",
3869 "data_digest"
3870 ],
3871 "truncate_seq": 0,
3872 "truncate_size": 0,
3873 "data_digest": "0x2ddbf8f5",
3874 "omap_digest": "0xffffffff",
3875 "expected_object_size": 0,
3876 "expected_write_size": 0,
3877 "alloc_hint_flags": 0,
3878 "manifest": {
3879 "type": 0
3880 },
3881 "watchers": {}
3882 },
3883 "union_shard_errors": [
3884 "missing"
3885 ],
3886 "errors": [],
3887 "object": {
3888 "version": 3,
3889 "snap": "head",
3890 "locator": "",
3891 "nspace": "",
3892 "name": "EOBJ3"
3893 }
3894 },
3895 {
3896 "shards": [
3897 {
3898 "attrs": [
3899 {
3900 "Base64": false,
3901 "value": "bad-val",
3902 "name": "key1-EOBJ4"
3903 },
3904 {
3905 "Base64": false,
3906 "value": "val2-EOBJ4",
3907 "name": "key2-EOBJ4"
3908 }
3909 ],
3910 "size": 2048,
3911 "errors": [],
3912 "shard": 2,
3913 "osd": 0,
3914 "primary": false
3915 },
3916 {
3917 "osd": 1,
3918 "primary": true,
3919 "shard": 0,
3920 "errors": [],
3921 "size": 2048,
3922 "attrs": [
3923 {
3924 "Base64": false,
3925 "value": "val1-EOBJ4",
3926 "name": "key1-EOBJ4"
3927 },
3928 {
3929 "Base64": false,
3930 "value": "val2-EOBJ4",
3931 "name": "key2-EOBJ4"
3932 }
3933 ]
3934 },
3935 {
3936 "osd": 2,
3937 "primary": false,
3938 "shard": 1,
3939 "errors": [],
3940 "size": 2048,
3941 "attrs": [
3942 {
3943 "Base64": false,
3944 "value": "val1-EOBJ4",
3945 "name": "key1-EOBJ4"
3946 },
3947 {
3948 "Base64": false,
3949 "value": "val3-EOBJ4",
3950 "name": "key3-EOBJ4"
3951 }
3952 ]
3953 }
3954 ],
3955 "selected_object_info": {
3956 "oid": {
3957 "oid": "EOBJ4",
3958 "key": "",
3959 "snapid": -2,
3960 "hash": 1618759290,
3961 "max": 0,
3962 "pool": 3,
3963 "namespace": ""
3964 },
3965 "version": "45'6",
3966 "prior_version": "45'5",
3967 "last_reqid": "client.4294.0:1",
3968 "user_version": 6,
3969 "size": 7,
3970 "mtime": "",
3971 "local_mtime": "",
3972 "lost": 0,
3973 "flags": [
3974 "dirty",
3975 "data_digest"
3976 ],
3977 "truncate_seq": 0,
3978 "truncate_size": 0,
3979 "data_digest": "0x2ddbf8f5",
3980 "omap_digest": "0xffffffff",
3981 "expected_object_size": 0,
3982 "expected_write_size": 0,
3983 "alloc_hint_flags": 0,
3984 "manifest": {
3985 "type": 0
3986 },
3987 "watchers": {}
3988 },
3989 "union_shard_errors": [],
3990 "errors": [
3991 "attr_value_mismatch",
3992 "attr_name_mismatch"
3993 ],
3994 "object": {
3995 "version": 6,
3996 "snap": "head",
3997 "locator": "",
3998 "nspace": "",
3999 "name": "EOBJ4"
4000 }
4001 },
4002 {
4003 "shards": [
4004 {
4005 "size": 2048,
4006 "errors": [],
4007 "shard": 2,
4008 "osd": 0,
4009 "primary": false
4010 },
4011 {
4012 "object_info": {
4013 "oid": {
4014 "oid": "EOBJ5",
4015 "key": "",
4016 "snapid": -2,
4017 "hash": 2918945441,
4018 "max": 0,
4019 "pool": 3,
4020 "namespace": ""
4021 },
4022 "version": "59'7",
4023 "prior_version": "0'0",
4024 "last_reqid": "client.4382.0:1",
4025 "user_version": 7,
4026 "size": 7,
4027 "mtime": "",
4028 "local_mtime": "",
4029 "lost": 0,
4030 "flags": [
4031 "dirty",
4032 "data_digest"
4033 ],
4034 "truncate_seq": 0,
4035 "truncate_size": 0,
4036 "data_digest": "0x2ddbf8f5",
4037 "omap_digest": "0xffffffff",
4038 "expected_object_size": 0,
4039 "expected_write_size": 0,
4040 "alloc_hint_flags": 0,
4041 "manifest": {
4042 "type": 0
4043 },
4044 "watchers": {}
4045 },
4046 "size": 4096,
4047 "shard": 0,
4048 "errors": [
4049 "size_mismatch_info",
4050 "obj_size_info_mismatch"
4051 ],
4052 "osd": 1,
4053 "primary": true
4054 },
4055 {
4056 "size": 2048,
4057 "shard": 1,
4058 "errors": [],
4059 "osd": 2,
4060 "primary": false
4061 }
4062 ],
4063 "selected_object_info": {
4064 "oid": {
4065 "oid": "EOBJ5",
4066 "key": "",
4067 "snapid": -2,
4068 "hash": 2918945441,
4069 "max": 0,
4070 "pool": 3,
4071 "namespace": ""
4072 },
4073 "version": "59'7",
4074 "prior_version": "0'0",
4075 "last_reqid": "client.4382.0:1",
4076 "user_version": 7,
4077 "size": 7,
4078 "mtime": "",
4079 "local_mtime": "",
4080 "lost": 0,
4081 "flags": [
4082 "dirty",
4083 "data_digest"
4084 ],
4085 "truncate_seq": 0,
4086 "truncate_size": 0,
4087 "data_digest": "0x2ddbf8f5",
4088 "omap_digest": "0xffffffff",
4089 "expected_object_size": 0,
4090 "expected_write_size": 0,
4091 "alloc_hint_flags": 0,
4092 "manifest": {
4093 "type": 0
4094 },
4095 "watchers": {}
4096 },
4097 "union_shard_errors": [
4098 "size_mismatch_info",
4099 "obj_size_info_mismatch"
4100 ],
4101 "errors": [
4102 "size_mismatch"
4103 ],
4104 "object": {
4105 "version": 7,
4106 "snap": "head",
4107 "locator": "",
4108 "nspace": "",
4109 "name": "EOBJ5"
4110 }
4111 },
4112 {
4113 "errors": [],
4114 "object": {
4115 "locator": "",
4116 "name": "EOBJ6",
4117 "nspace": "",
4118 "snap": "head",
4119 "version": 8
4120 },
4121 "selected_object_info": {
4122 "oid": {
4123 "oid": "EOBJ6",
4124 "key": "",
4125 "snapid": -2,
4126 "hash": 3050890866,
4127 "max": 0,
4128 "pool": 3,
4129 "namespace": ""
4130 },
4131 "version": "65'8",
4132 "prior_version": "0'0",
4133 "last_reqid": "client.4418.0:1",
4134 "user_version": 8,
4135 "size": 7,
4136 "mtime": "",
4137 "local_mtime": "",
4138 "lost": 0,
4139 "flags": [
4140 "dirty",
4141 "data_digest"
4142 ],
4143 "truncate_seq": 0,
4144 "truncate_size": 0,
4145 "data_digest": "0x2ddbf8f5",
4146 "omap_digest": "0xffffffff",
4147 "expected_object_size": 0,
4148 "expected_write_size": 0,
4149 "alloc_hint_flags": 0,
4150 "manifest": {
4151 "type": 0
4152 },
4153 "watchers": {}
4154 },
4155 "shards": [
4156 {
4157 "errors": [
4158 "hinfo_missing"
4159 ],
4160 "osd": 0,
4161 "primary": false,
4162 "shard": 2,
4163 "size": 2048
4164 },
4165 {
4166 "errors": [
4167 "hinfo_corrupted"
4168 ],
4169 "osd": 1,
4170 "primary": true,
4171 "shard": 0,
4172 "hashinfo": "bad-val",
4173 "size": 2048
4174 },
4175 {
4176 "errors": [],
4177 "osd": 2,
4178 "primary": false,
4179 "shard": 1,
4180 "size": 2048,
4181 "hashinfo": {
4182 "cumulative_shard_hashes": [
4183 {
4184 "hash": 80717615,
4185 "shard": 0
4186 },
4187 {
4188 "hash": 1534491824,
4189 "shard": 1
4190 },
4191 {
4192 "hash": 80717615,
4193 "shard": 2
4194 }
4195 ],
4196 "total_chunk_size": 2048
4197 }
4198 }
4199 ],
4200 "union_shard_errors": [
4201 "hinfo_missing",
4202 "hinfo_corrupted"
4203 ]
4204 },
4205 {
4206 "errors": [
4207 "hinfo_inconsistency"
4208 ],
4209 "object": {
4210 "locator": "",
4211 "name": "EOBJ7",
4212 "nspace": "",
4213 "snap": "head",
4214 "version": 10
4215 },
4216 "selected_object_info": {
4217 "oid": {
4218 "oid": "EOBJ7",
4219 "key": "",
4220 "snapid": -2,
4221 "hash": 3258066308,
4222 "max": 0,
4223 "pool": 3,
4224 "namespace": ""
4225 },
4226 "version": "75'10",
4227 "prior_version": "75'9",
4228 "last_reqid": "client.4482.0:1",
4229 "user_version": 10,
4230 "size": 34,
4231 "mtime": "",
4232 "local_mtime": "",
4233 "lost": 0,
4234 "flags": [
4235 "dirty",
4236 "data_digest"
4237 ],
4238 "truncate_seq": 0,
4239 "truncate_size": 0,
4240 "data_digest": "0x136e4e27",
4241 "omap_digest": "0xffffffff",
4242 "expected_object_size": 0,
4243 "expected_write_size": 0,
4244 "alloc_hint_flags": 0,
4245 "manifest": {
4246 "type": 0
4247 },
4248 "watchers": {}
4249 },
4250 "shards": [
4251 {
4252 "hashinfo": {
4253 "cumulative_shard_hashes": [
4254 {
4255 "hash": 80717615,
4256 "shard": 0
4257 },
4258 {
4259 "hash": 1534491824,
4260 "shard": 1
4261 },
4262 {
4263 "hash": 80717615,
4264 "shard": 2
4265 }
4266 ],
4267 "total_chunk_size": 2048
4268 },
4269 "errors": [],
4270 "osd": 0,
4271 "primary": false,
4272 "shard": 2,
4273 "size": 2048
4274 },
4275 {
4276 "hashinfo": {
4277 "cumulative_shard_hashes": [
4278 {
4279 "hash": 1534350760,
4280 "shard": 0
4281 },
4282 {
4283 "hash": 1534491824,
4284 "shard": 1
4285 },
4286 {
4287 "hash": 1534350760,
4288 "shard": 2
4289 }
4290 ],
4291 "total_chunk_size": 2048
4292 },
4293 "errors": [],
4294 "osd": 1,
4295 "primary": true,
4296 "shard": 0,
4297 "size": 2048
4298 },
4299 {
4300 "hashinfo": {
4301 "cumulative_shard_hashes": [
4302 {
4303 "hash": 1534350760,
4304 "shard": 0
4305 },
4306 {
4307 "hash": 1534491824,
4308 "shard": 1
4309 },
4310 {
4311 "hash": 1534350760,
4312 "shard": 2
4313 }
4314 ],
4315 "total_chunk_size": 2048
4316 },
4317 "errors": [],
4318 "osd": 2,
4319 "primary": false,
4320 "shard": 1,
4321 "size": 2048
4322 }
4323 ],
4324 "union_shard_errors": []
4325 }
4326 ],
4327 "epoch": 0
4328 }
4329 EOF
4330
4331 jq "$jqfilter" $dir/json | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/csjson
4332 multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
4333 if test $getjson = "yes"
4334 then
4335 jq '.' $dir/json > save3.json
4336 fi
4337
4338 if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
4339 then
4340 jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
4341 fi
4342
4343 pg_deep_scrub $pg
4344
4345 rados list-inconsistent-pg $poolname > $dir/json || return 1
4346 # Check pg count
4347 test $(jq '. | length' $dir/json) = "1" || return 1
4348 # Check pgid
4349 test $(jq -r '.[0]' $dir/json) = $pg || return 1
4350
4351 rados list-inconsistent-obj $pg > $dir/json || return 1
4352 # Get epoch for repair-get requests
4353 epoch=$(jq .epoch $dir/json)
4354
4355 if [ "$allow_overwrites" = "true" ]
4356 then
4357 jq "$jqfilter" << EOF | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/checkcsjson
4358 {
4359 "inconsistents": [
4360 {
4361 "shards": [
4362 {
4363 "data_digest": "0x00000000",
4364 "omap_digest": "0xffffffff",
4365 "size": 2048,
4366 "errors": [],
4367 "shard": 2,
4368 "osd": 0,
4369 "primary": false
4370 },
4371 {
4372 "object_info": {
4373 "oid": {
4374 "oid": "EOBJ1",
4375 "key": "",
4376 "snapid": -2,
4377 "hash": 560836233,
4378 "max": 0,
4379 "pool": 3,
4380 "namespace": ""
4381 },
4382 "version": "27'1",
4383 "prior_version": "0'0",
4384 "last_reqid": "client.4184.0:1",
4385 "user_version": 1,
4386 "size": 7,
4387 "mtime": "2018-04-05 14:31:33.837147",
4388 "local_mtime": "2018-04-05 14:31:33.840763",
4389 "lost": 0,
4390 "flags": [
4391 "dirty",
4392 "data_digest"
4393 ],
4394 "truncate_seq": 0,
4395 "truncate_size": 0,
4396 "data_digest": "0x2ddbf8f5",
4397 "omap_digest": "0xffffffff",
4398 "expected_object_size": 0,
4399 "expected_write_size": 0,
4400 "alloc_hint_flags": 0,
4401 "manifest": {
4402 "type": 0
4403 },
4404 "watchers": {}
4405 },
4406 "size": 9,
4407 "shard": 0,
4408 "errors": [
4409 "read_error",
4410 "size_mismatch_info",
4411 "obj_size_info_mismatch"
4412 ],
4413 "osd": 1,
4414 "primary": true
4415 },
4416 {
4417 "data_digest": "0x00000000",
4418 "omap_digest": "0xffffffff",
4419 "size": 2048,
4420 "shard": 1,
4421 "errors": [],
4422 "osd": 2,
4423 "primary": false
4424 }
4425 ],
4426 "selected_object_info": {
4427 "oid": {
4428 "oid": "EOBJ1",
4429 "key": "",
4430 "snapid": -2,
4431 "hash": 560836233,
4432 "max": 0,
4433 "pool": 3,
4434 "namespace": ""
4435 },
4436 "version": "27'1",
4437 "prior_version": "0'0",
4438 "last_reqid": "client.4184.0:1",
4439 "user_version": 1,
4440 "size": 7,
4441 "mtime": "2018-04-05 14:31:33.837147",
4442 "local_mtime": "2018-04-05 14:31:33.840763",
4443 "lost": 0,
4444 "flags": [
4445 "dirty",
4446 "data_digest"
4447 ],
4448 "truncate_seq": 0,
4449 "truncate_size": 0,
4450 "data_digest": "0x2ddbf8f5",
4451 "omap_digest": "0xffffffff",
4452 "expected_object_size": 0,
4453 "expected_write_size": 0,
4454 "alloc_hint_flags": 0,
4455 "manifest": {
4456 "type": 0
4457 },
4458 "watchers": {}
4459 },
4460 "union_shard_errors": [
4461 "read_error",
4462 "size_mismatch_info",
4463 "obj_size_info_mismatch"
4464 ],
4465 "errors": [
4466 "size_mismatch"
4467 ],
4468 "object": {
4469 "version": 1,
4470 "snap": "head",
4471 "locator": "",
4472 "nspace": "",
4473 "name": "EOBJ1"
4474 }
4475 },
4476 {
4477 "shards": [
4478 {
4479 "data_digest": "0x00000000",
4480 "omap_digest": "0xffffffff",
4481 "size": 2048,
4482 "errors": [],
4483 "shard": 2,
4484 "osd": 0,
4485 "primary": false
4486 },
4487 {
4488 "shard": 0,
4489 "errors": [
4490 "missing"
4491 ],
4492 "osd": 1,
4493 "primary": true
4494 },
4495 {
4496 "data_digest": "0x00000000",
4497 "omap_digest": "0xffffffff",
4498 "size": 2048,
4499 "shard": 1,
4500 "errors": [],
4501 "osd": 2,
4502 "primary": false
4503 }
4504 ],
4505 "selected_object_info": {
4506 "oid": {
4507 "oid": "EOBJ3",
4508 "key": "",
4509 "snapid": -2,
4510 "hash": 3125668237,
4511 "max": 0,
4512 "pool": 3,
4513 "namespace": ""
4514 },
4515 "version": "39'3",
4516 "prior_version": "0'0",
4517 "last_reqid": "client.4252.0:1",
4518 "user_version": 3,
4519 "size": 7,
4520 "mtime": "2018-04-05 14:31:46.841145",
4521 "local_mtime": "2018-04-05 14:31:46.844996",
4522 "lost": 0,
4523 "flags": [
4524 "dirty",
4525 "data_digest"
4526 ],
4527 "truncate_seq": 0,
4528 "truncate_size": 0,
4529 "data_digest": "0x2ddbf8f5",
4530 "omap_digest": "0xffffffff",
4531 "expected_object_size": 0,
4532 "expected_write_size": 0,
4533 "alloc_hint_flags": 0,
4534 "manifest": {
4535 "type": 0
4536 },
4537 "watchers": {}
4538 },
4539 "union_shard_errors": [
4540 "missing"
4541 ],
4542 "errors": [],
4543 "object": {
4544 "version": 3,
4545 "snap": "head",
4546 "locator": "",
4547 "nspace": "",
4548 "name": "EOBJ3"
4549 }
4550 },
4551 {
4552 "shards": [
4553 {
4554 "attrs": [
4555 {
4556 "Base64": false,
4557 "value": "bad-val",
4558 "name": "key1-EOBJ4"
4559 },
4560 {
4561 "Base64": false,
4562 "value": "val2-EOBJ4",
4563 "name": "key2-EOBJ4"
4564 }
4565 ],
4566 "data_digest": "0x00000000",
4567 "omap_digest": "0xffffffff",
4568 "size": 2048,
4569 "errors": [],
4570 "shard": 2,
4571 "osd": 0,
4572 "primary": false
4573 },
4574 {
4575 "attrs": [
4576 {
4577 "Base64": false,
4578 "value": "val1-EOBJ4",
4579 "name": "key1-EOBJ4"
4580 },
4581 {
4582 "Base64": false,
4583 "value": "val2-EOBJ4",
4584 "name": "key2-EOBJ4"
4585 }
4586 ],
4587 "data_digest": "0x00000000",
4588 "omap_digest": "0xffffffff",
4589 "size": 2048,
4590 "errors": [],
4591 "shard": 0,
4592 "osd": 1,
4593 "primary": true
4594 },
4595 {
4596 "attrs": [
4597 {
4598 "Base64": false,
4599 "value": "val1-EOBJ4",
4600 "name": "key1-EOBJ4"
4601 },
4602 {
4603 "Base64": false,
4604 "value": "val3-EOBJ4",
4605 "name": "key3-EOBJ4"
4606 }
4607 ],
4608 "data_digest": "0x00000000",
4609 "omap_digest": "0xffffffff",
4610 "size": 2048,
4611 "errors": [],
4612 "shard": 1,
4613 "osd": 2,
4614 "primary": false
4615 }
4616 ],
4617 "selected_object_info": {
4618 "oid": {
4619 "oid": "EOBJ4",
4620 "key": "",
4621 "snapid": -2,
4622 "hash": 1618759290,
4623 "max": 0,
4624 "pool": 3,
4625 "namespace": ""
4626 },
4627 "version": "45'6",
4628 "prior_version": "45'5",
4629 "last_reqid": "client.4294.0:1",
4630 "user_version": 6,
4631 "size": 7,
4632 "mtime": "2018-04-05 14:31:54.663622",
4633 "local_mtime": "2018-04-05 14:31:54.664527",
4634 "lost": 0,
4635 "flags": [
4636 "dirty",
4637 "data_digest"
4638 ],
4639 "truncate_seq": 0,
4640 "truncate_size": 0,
4641 "data_digest": "0x2ddbf8f5",
4642 "omap_digest": "0xffffffff",
4643 "expected_object_size": 0,
4644 "expected_write_size": 0,
4645 "alloc_hint_flags": 0,
4646 "manifest": {
4647 "type": 0
4648 },
4649 "watchers": {}
4650 },
4651 "union_shard_errors": [],
4652 "errors": [
4653 "attr_value_mismatch",
4654 "attr_name_mismatch"
4655 ],
4656 "object": {
4657 "version": 6,
4658 "snap": "head",
4659 "locator": "",
4660 "nspace": "",
4661 "name": "EOBJ4"
4662 }
4663 },
4664 {
4665 "shards": [
4666 {
4667 "data_digest": "0x00000000",
4668 "omap_digest": "0xffffffff",
4669 "size": 2048,
4670 "errors": [],
4671 "shard": 2,
4672 "osd": 0,
4673 "primary": false
4674 },
4675 {
4676 "object_info": {
4677 "oid": {
4678 "oid": "EOBJ5",
4679 "key": "",
4680 "snapid": -2,
4681 "hash": 2918945441,
4682 "max": 0,
4683 "pool": 3,
4684 "namespace": ""
4685 },
4686 "version": "59'7",
4687 "prior_version": "0'0",
4688 "last_reqid": "client.4382.0:1",
4689 "user_version": 7,
4690 "size": 7,
4691 "mtime": "2018-04-05 14:32:12.929161",
4692 "local_mtime": "2018-04-05 14:32:12.934707",
4693 "lost": 0,
4694 "flags": [
4695 "dirty",
4696 "data_digest"
4697 ],
4698 "truncate_seq": 0,
4699 "truncate_size": 0,
4700 "data_digest": "0x2ddbf8f5",
4701 "omap_digest": "0xffffffff",
4702 "expected_object_size": 0,
4703 "expected_write_size": 0,
4704 "alloc_hint_flags": 0,
4705 "manifest": {
4706 "type": 0
4707 },
4708 "watchers": {}
4709 },
4710 "size": 4096,
4711 "errors": [
4712 "read_error",
4713 "size_mismatch_info",
4714 "obj_size_info_mismatch"
4715 ],
4716 "shard": 0,
4717 "osd": 1,
4718 "primary": true
4719 },
4720 {
4721 "data_digest": "0x00000000",
4722 "omap_digest": "0xffffffff",
4723 "size": 2048,
4724 "errors": [],
4725 "shard": 1,
4726 "osd": 2,
4727 "primary": false
4728 }
4729 ],
4730 "selected_object_info": {
4731 "oid": {
4732 "oid": "EOBJ5",
4733 "key": "",
4734 "snapid": -2,
4735 "hash": 2918945441,
4736 "max": 0,
4737 "pool": 3,
4738 "namespace": ""
4739 },
4740 "version": "59'7",
4741 "prior_version": "0'0",
4742 "last_reqid": "client.4382.0:1",
4743 "user_version": 7,
4744 "size": 7,
4745 "mtime": "2018-04-05 14:32:12.929161",
4746 "local_mtime": "2018-04-05 14:32:12.934707",
4747 "lost": 0,
4748 "flags": [
4749 "dirty",
4750 "data_digest"
4751 ],
4752 "truncate_seq": 0,
4753 "truncate_size": 0,
4754 "data_digest": "0x2ddbf8f5",
4755 "omap_digest": "0xffffffff",
4756 "expected_object_size": 0,
4757 "expected_write_size": 0,
4758 "alloc_hint_flags": 0,
4759 "manifest": {
4760 "type": 0
4761 },
4762 "watchers": {}
4763 },
4764 "union_shard_errors": [
4765 "read_error",
4766 "size_mismatch_info",
4767 "obj_size_info_mismatch"
4768 ],
4769 "errors": [
4770 "size_mismatch"
4771 ],
4772 "object": {
4773 "version": 7,
4774 "snap": "head",
4775 "locator": "",
4776 "nspace": "",
4777 "name": "EOBJ5"
4778 }
4779 },
4780 {
4781 "object": {
4782 "name": "EOBJ6",
4783 "nspace": "",
4784 "locator": "",
4785 "snap": "head",
4786 "version": 8
4787 },
4788 "errors": [],
4789 "union_shard_errors": [
4790 "read_error",
4791 "hinfo_missing",
4792 "hinfo_corrupted"
4793 ],
4794 "selected_object_info": {
4795 "oid": {
4796 "oid": "EOBJ6",
4797 "key": "",
4798 "snapid": -2,
4799 "hash": 3050890866,
4800 "max": 0,
4801 "pool": 3,
4802 "namespace": ""
4803 },
4804 "version": "65'8",
4805 "prior_version": "0'0",
4806 "last_reqid": "client.4418.0:1",
4807 "user_version": 8,
4808 "size": 7,
4809 "mtime": "2018-04-05 14:32:20.634116",
4810 "local_mtime": "2018-04-05 14:32:20.637999",
4811 "lost": 0,
4812 "flags": [
4813 "dirty",
4814 "data_digest"
4815 ],
4816 "truncate_seq": 0,
4817 "truncate_size": 0,
4818 "data_digest": "0x2ddbf8f5",
4819 "omap_digest": "0xffffffff",
4820 "expected_object_size": 0,
4821 "expected_write_size": 0,
4822 "alloc_hint_flags": 0,
4823 "manifest": {
4824 "type": 0
4825 },
4826 "watchers": {}
4827 },
4828 "shards": [
4829 {
4830 "osd": 0,
4831 "primary": false,
4832 "shard": 2,
4833 "errors": [
4834 "read_error",
4835 "hinfo_missing"
4836 ],
4837 "size": 2048
4838 },
4839 {
4840 "osd": 1,
4841 "primary": true,
4842 "shard": 0,
4843 "errors": [
4844 "read_error",
4845 "hinfo_corrupted"
4846 ],
4847 "size": 2048,
4848 "hashinfo": "bad-val"
4849 },
4850 {
4851 "osd": 2,
4852 "primary": false,
4853 "shard": 1,
4854 "errors": [],
4855 "size": 2048,
4856 "omap_digest": "0xffffffff",
4857 "data_digest": "0x00000000",
4858 "hashinfo": {
4859 "cumulative_shard_hashes": [
4860 {
4861 "hash": 80717615,
4862 "shard": 0
4863 },
4864 {
4865 "hash": 1534491824,
4866 "shard": 1
4867 },
4868 {
4869 "hash": 80717615,
4870 "shard": 2
4871 }
4872 ],
4873 "total_chunk_size": 2048
4874 }
4875 }
4876 ]
4877 },
4878 {
4879 "object": {
4880 "name": "EOBJ7",
4881 "nspace": "",
4882 "locator": "",
4883 "snap": "head",
4884 "version": 10
4885 },
4886 "errors": [
4887 "hinfo_inconsistency"
4888 ],
4889 "union_shard_errors": [],
4890 "selected_object_info": {
4891 "oid": {
4892 "oid": "EOBJ7",
4893 "key": "",
4894 "snapid": -2,
4895 "hash": 3258066308,
4896 "max": 0,
4897 "pool": 3,
4898 "namespace": ""
4899 },
4900 "version": "75'10",
4901 "prior_version": "75'9",
4902 "last_reqid": "client.4482.0:1",
4903 "user_version": 10,
4904 "size": 34,
4905 "mtime": "2018-04-05 14:32:33.058782",
4906 "local_mtime": "2018-04-05 14:32:33.059679",
4907 "lost": 0,
4908 "flags": [
4909 "dirty",
4910 "data_digest"
4911 ],
4912 "truncate_seq": 0,
4913 "truncate_size": 0,
4914 "data_digest": "0x136e4e27",
4915 "omap_digest": "0xffffffff",
4916 "expected_object_size": 0,
4917 "expected_write_size": 0,
4918 "alloc_hint_flags": 0,
4919 "manifest": {
4920 "type": 0
4921 },
4922 "watchers": {}
4923 },
4924 "shards": [
4925 {
4926 "osd": 0,
4927 "primary": false,
4928 "shard": 2,
4929 "errors": [],
4930 "size": 2048,
4931 "omap_digest": "0xffffffff",
4932 "data_digest": "0x00000000",
4933 "hashinfo": {
4934 "cumulative_shard_hashes": [
4935 {
4936 "hash": 80717615,
4937 "shard": 0
4938 },
4939 {
4940 "hash": 1534491824,
4941 "shard": 1
4942 },
4943 {
4944 "hash": 80717615,
4945 "shard": 2
4946 }
4947 ],
4948 "total_chunk_size": 2048
4949 }
4950 },
4951 {
4952 "osd": 1,
4953 "primary": true,
4954 "shard": 0,
4955 "errors": [],
4956 "size": 2048,
4957 "omap_digest": "0xffffffff",
4958 "data_digest": "0x00000000",
4959 "hashinfo": {
4960 "cumulative_shard_hashes": [
4961 {
4962 "hash": 1534350760,
4963 "shard": 0
4964 },
4965 {
4966 "hash": 1534491824,
4967 "shard": 1
4968 },
4969 {
4970 "hash": 1534350760,
4971 "shard": 2
4972 }
4973 ],
4974 "total_chunk_size": 2048
4975 }
4976 },
4977 {
4978 "osd": 2,
4979 "primary": false,
4980 "shard": 1,
4981 "errors": [],
4982 "size": 2048,
4983 "omap_digest": "0xffffffff",
4984 "data_digest": "0x00000000",
4985 "hashinfo": {
4986 "cumulative_shard_hashes": [
4987 {
4988 "hash": 1534350760,
4989 "shard": 0
4990 },
4991 {
4992 "hash": 1534491824,
4993 "shard": 1
4994 },
4995 {
4996 "hash": 1534350760,
4997 "shard": 2
4998 }
4999 ],
5000 "total_chunk_size": 2048
5001 }
5002 }
5003 ]
5004 }
5005 ],
5006 "epoch": 0
5007 }
5008 EOF
5009
5010 else
5011
5012 jq "$jqfilter" << EOF | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/checkcsjson
5013 {
5014 "inconsistents": [
5015 {
5016 "shards": [
5017 {
5018 "data_digest": "0x04cfa72f",
5019 "omap_digest": "0xffffffff",
5020 "size": 2048,
5021 "errors": [],
5022 "shard": 2,
5023 "osd": 0,
5024 "primary": false
5025 },
5026 {
5027 "object_info": {
5028 "oid": {
5029 "oid": "EOBJ1",
5030 "key": "",
5031 "snapid": -2,
5032 "hash": 560836233,
5033 "max": 0,
5034 "pool": 3,
5035 "namespace": ""
5036 },
5037 "version": "27'1",
5038 "prior_version": "0'0",
5039 "last_reqid": "client.4192.0:1",
5040 "user_version": 1,
5041 "size": 7,
5042 "mtime": "2018-04-05 14:30:10.688009",
5043 "local_mtime": "2018-04-05 14:30:10.691774",
5044 "lost": 0,
5045 "flags": [
5046 "dirty",
5047 "data_digest"
5048 ],
5049 "truncate_seq": 0,
5050 "truncate_size": 0,
5051 "data_digest": "0x2ddbf8f5",
5052 "omap_digest": "0xffffffff",
5053 "expected_object_size": 0,
5054 "expected_write_size": 0,
5055 "alloc_hint_flags": 0,
5056 "manifest": {
5057 "type": 0
5058 },
5059 "watchers": {}
5060 },
5061 "size": 9,
5062 "shard": 0,
5063 "errors": [
5064 "read_error",
5065 "size_mismatch_info",
5066 "obj_size_info_mismatch"
5067 ],
5068 "osd": 1,
5069 "primary": true
5070 },
5071 {
5072 "data_digest": "0x04cfa72f",
5073 "omap_digest": "0xffffffff",
5074 "size": 2048,
5075 "shard": 1,
5076 "errors": [],
5077 "osd": 2,
5078 "primary": false
5079 }
5080 ],
5081 "selected_object_info": {
5082 "oid": {
5083 "oid": "EOBJ1",
5084 "key": "",
5085 "snapid": -2,
5086 "hash": 560836233,
5087 "max": 0,
5088 "pool": 3,
5089 "namespace": ""
5090 },
5091 "version": "27'1",
5092 "prior_version": "0'0",
5093 "last_reqid": "client.4192.0:1",
5094 "user_version": 1,
5095 "size": 7,
5096 "mtime": "2018-04-05 14:30:10.688009",
5097 "local_mtime": "2018-04-05 14:30:10.691774",
5098 "lost": 0,
5099 "flags": [
5100 "dirty",
5101 "data_digest"
5102 ],
5103 "truncate_seq": 0,
5104 "truncate_size": 0,
5105 "data_digest": "0x2ddbf8f5",
5106 "omap_digest": "0xffffffff",
5107 "expected_object_size": 0,
5108 "expected_write_size": 0,
5109 "alloc_hint_flags": 0,
5110 "manifest": {
5111 "type": 0
5112 },
5113 "watchers": {}
5114 },
5115 "union_shard_errors": [
5116 "read_error",
5117 "size_mismatch_info",
5118 "obj_size_info_mismatch"
5119 ],
5120 "errors": [
5121 "size_mismatch"
5122 ],
5123 "object": {
5124 "version": 1,
5125 "snap": "head",
5126 "locator": "",
5127 "nspace": "",
5128 "name": "EOBJ1"
5129 }
5130 },
5131 {
5132 "shards": [
5133 {
5134 "size": 2048,
5135 "errors": [
5136 "ec_hash_error"
5137 ],
5138 "shard": 2,
5139 "osd": 0,
5140 "primary": false
5141 },
5142 {
5143 "data_digest": "0x04cfa72f",
5144 "omap_digest": "0xffffffff",
5145 "size": 2048,
5146 "errors": [],
5147 "shard": 0,
5148 "osd": 1,
5149 "primary": true
5150 },
5151 {
5152 "data_digest": "0x04cfa72f",
5153 "omap_digest": "0xffffffff",
5154 "size": 2048,
5155 "errors": [],
5156 "shard": 1,
5157 "osd": 2,
5158 "primary": false
5159 }
5160 ],
5161 "selected_object_info": {
5162 "oid": {
5163 "oid": "EOBJ2",
5164 "key": "",
5165 "snapid": -2,
5166 "hash": 562812377,
5167 "max": 0,
5168 "pool": 3,
5169 "namespace": ""
5170 },
5171 "version": "33'2",
5172 "prior_version": "0'0",
5173 "last_reqid": "client.4224.0:1",
5174 "user_version": 2,
5175 "size": 7,
5176 "mtime": "2018-04-05 14:30:14.152945",
5177 "local_mtime": "2018-04-05 14:30:14.154014",
5178 "lost": 0,
5179 "flags": [
5180 "dirty",
5181 "data_digest"
5182 ],
5183 "truncate_seq": 0,
5184 "truncate_size": 0,
5185 "data_digest": "0x2ddbf8f5",
5186 "omap_digest": "0xffffffff",
5187 "expected_object_size": 0,
5188 "expected_write_size": 0,
5189 "alloc_hint_flags": 0,
5190 "manifest": {
5191 "type": 0
5192 },
5193 "watchers": {}
5194 },
5195 "union_shard_errors": [
5196 "ec_hash_error"
5197 ],
5198 "errors": [],
5199 "object": {
5200 "version": 2,
5201 "snap": "head",
5202 "locator": "",
5203 "nspace": "",
5204 "name": "EOBJ2"
5205 }
5206 },
5207 {
5208 "shards": [
5209 {
5210 "data_digest": "0x04cfa72f",
5211 "omap_digest": "0xffffffff",
5212 "size": 2048,
5213 "errors": [],
5214 "shard": 2,
5215 "osd": 0,
5216 "primary": false
5217 },
5218 {
5219 "osd": 1,
5220 "primary": true,
5221 "shard": 0,
5222 "errors": [
5223 "missing"
5224 ]
5225 },
5226 {
5227 "data_digest": "0x04cfa72f",
5228 "omap_digest": "0xffffffff",
5229 "size": 2048,
5230 "shard": 1,
5231 "errors": [],
5232 "osd": 2,
5233 "primary": false
5234 }
5235 ],
5236 "selected_object_info": {
5237 "oid": {
5238 "oid": "EOBJ3",
5239 "key": "",
5240 "snapid": -2,
5241 "hash": 3125668237,
5242 "max": 0,
5243 "pool": 3,
5244 "namespace": ""
5245 },
5246 "version": "39'3",
5247 "prior_version": "0'0",
5248 "last_reqid": "client.4258.0:1",
5249 "user_version": 3,
5250 "size": 7,
5251 "mtime": "2018-04-05 14:30:18.875544",
5252 "local_mtime": "2018-04-05 14:30:18.880153",
5253 "lost": 0,
5254 "flags": [
5255 "dirty",
5256 "data_digest"
5257 ],
5258 "truncate_seq": 0,
5259 "truncate_size": 0,
5260 "data_digest": "0x2ddbf8f5",
5261 "omap_digest": "0xffffffff",
5262 "expected_object_size": 0,
5263 "expected_write_size": 0,
5264 "alloc_hint_flags": 0,
5265 "manifest": {
5266 "type": 0
5267 },
5268 "watchers": {}
5269 },
5270 "union_shard_errors": [
5271 "missing"
5272 ],
5273 "errors": [],
5274 "object": {
5275 "version": 3,
5276 "snap": "head",
5277 "locator": "",
5278 "nspace": "",
5279 "name": "EOBJ3"
5280 }
5281 },
5282 {
5283 "shards": [
5284 {
5285 "attrs": [
5286 {
5287 "Base64": false,
5288 "value": "bad-val",
5289 "name": "key1-EOBJ4"
5290 },
5291 {
5292 "Base64": false,
5293 "value": "val2-EOBJ4",
5294 "name": "key2-EOBJ4"
5295 }
5296 ],
5297 "data_digest": "0x04cfa72f",
5298 "omap_digest": "0xffffffff",
5299 "size": 2048,
5300 "errors": [],
5301 "shard": 2,
5302 "osd": 0,
5303 "primary": false
5304 },
5305 {
5306 "osd": 1,
5307 "primary": true,
5308 "shard": 0,
5309 "errors": [],
5310 "size": 2048,
5311 "omap_digest": "0xffffffff",
5312 "data_digest": "0x04cfa72f",
5313 "attrs": [
5314 {
5315 "Base64": false,
5316 "value": "val1-EOBJ4",
5317 "name": "key1-EOBJ4"
5318 },
5319 {
5320 "Base64": false,
5321 "value": "val2-EOBJ4",
5322 "name": "key2-EOBJ4"
5323 }
5324 ]
5325 },
5326 {
5327 "osd": 2,
5328 "primary": false,
5329 "shard": 1,
5330 "errors": [],
5331 "size": 2048,
5332 "omap_digest": "0xffffffff",
5333 "data_digest": "0x04cfa72f",
5334 "attrs": [
5335 {
5336 "Base64": false,
5337 "value": "val1-EOBJ4",
5338 "name": "key1-EOBJ4"
5339 },
5340 {
5341 "Base64": false,
5342 "value": "val3-EOBJ4",
5343 "name": "key3-EOBJ4"
5344 }
5345 ]
5346 }
5347 ],
5348 "selected_object_info": {
5349 "oid": {
5350 "oid": "EOBJ4",
5351 "key": "",
5352 "snapid": -2,
5353 "hash": 1618759290,
5354 "max": 0,
5355 "pool": 3,
5356 "namespace": ""
5357 },
5358 "version": "45'6",
5359 "prior_version": "45'5",
5360 "last_reqid": "client.4296.0:1",
5361 "user_version": 6,
5362 "size": 7,
5363 "mtime": "2018-04-05 14:30:22.271983",
5364 "local_mtime": "2018-04-05 14:30:22.272840",
5365 "lost": 0,
5366 "flags": [
5367 "dirty",
5368 "data_digest"
5369 ],
5370 "truncate_seq": 0,
5371 "truncate_size": 0,
5372 "data_digest": "0x2ddbf8f5",
5373 "omap_digest": "0xffffffff",
5374 "expected_object_size": 0,
5375 "expected_write_size": 0,
5376 "alloc_hint_flags": 0,
5377 "manifest": {
5378 "type": 0
5379 },
5380 "watchers": {}
5381 },
5382 "union_shard_errors": [],
5383 "errors": [
5384 "attr_value_mismatch",
5385 "attr_name_mismatch"
5386 ],
5387 "object": {
5388 "version": 6,
5389 "snap": "head",
5390 "locator": "",
5391 "nspace": "",
5392 "name": "EOBJ4"
5393 }
5394 },
5395 {
5396 "shards": [
5397 {
5398 "data_digest": "0x04cfa72f",
5399 "omap_digest": "0xffffffff",
5400 "size": 2048,
5401 "errors": [],
5402 "shard": 2,
5403 "osd": 0,
5404 "primary": false
5405 },
5406 {
5407 "object_info": {
5408 "oid": {
5409 "oid": "EOBJ5",
5410 "key": "",
5411 "snapid": -2,
5412 "hash": 2918945441,
5413 "max": 0,
5414 "pool": 3,
5415 "namespace": ""
5416 },
5417 "version": "59'7",
5418 "prior_version": "0'0",
5419 "last_reqid": "client.4384.0:1",
5420 "user_version": 7,
5421 "size": 7,
5422 "mtime": "2018-04-05 14:30:35.162395",
5423 "local_mtime": "2018-04-05 14:30:35.166390",
5424 "lost": 0,
5425 "flags": [
5426 "dirty",
5427 "data_digest"
5428 ],
5429 "truncate_seq": 0,
5430 "truncate_size": 0,
5431 "data_digest": "0x2ddbf8f5",
5432 "omap_digest": "0xffffffff",
5433 "expected_object_size": 0,
5434 "expected_write_size": 0,
5435 "alloc_hint_flags": 0,
5436 "manifest": {
5437 "type": 0
5438 },
5439 "watchers": {}
5440 },
5441 "size": 4096,
5442 "shard": 0,
5443 "errors": [
5444 "read_error",
5445 "size_mismatch_info",
5446 "obj_size_info_mismatch"
5447 ],
5448 "osd": 1,
5449 "primary": true
5450 },
5451 {
5452 "data_digest": "0x04cfa72f",
5453 "omap_digest": "0xffffffff",
5454 "size": 2048,
5455 "shard": 1,
5456 "errors": [],
5457 "osd": 2,
5458 "primary": false
5459 }
5460 ],
5461 "selected_object_info": {
5462 "oid": {
5463 "oid": "EOBJ5",
5464 "key": "",
5465 "snapid": -2,
5466 "hash": 2918945441,
5467 "max": 0,
5468 "pool": 3,
5469 "namespace": ""
5470 },
5471 "version": "59'7",
5472 "prior_version": "0'0",
5473 "last_reqid": "client.4384.0:1",
5474 "user_version": 7,
5475 "size": 7,
5476 "mtime": "2018-04-05 14:30:35.162395",
5477 "local_mtime": "2018-04-05 14:30:35.166390",
5478 "lost": 0,
5479 "flags": [
5480 "dirty",
5481 "data_digest"
5482 ],
5483 "truncate_seq": 0,
5484 "truncate_size": 0,
5485 "data_digest": "0x2ddbf8f5",
5486 "omap_digest": "0xffffffff",
5487 "expected_object_size": 0,
5488 "expected_write_size": 0,
5489 "alloc_hint_flags": 0,
5490 "manifest": {
5491 "type": 0
5492 },
5493 "watchers": {}
5494 },
5495 "union_shard_errors": [
5496 "read_error",
5497 "size_mismatch_info",
5498 "obj_size_info_mismatch"
5499 ],
5500 "errors": [
5501 "size_mismatch"
5502 ],
5503 "object": {
5504 "version": 7,
5505 "snap": "head",
5506 "locator": "",
5507 "nspace": "",
5508 "name": "EOBJ5"
5509 }
5510 },
5511 {
5512 "object": {
5513 "name": "EOBJ6",
5514 "nspace": "",
5515 "locator": "",
5516 "snap": "head",
5517 "version": 8
5518 },
5519 "errors": [],
5520 "union_shard_errors": [
5521 "read_error",
5522 "hinfo_missing",
5523 "hinfo_corrupted"
5524 ],
5525 "selected_object_info": {
5526 "oid": {
5527 "oid": "EOBJ6",
5528 "key": "",
5529 "snapid": -2,
5530 "hash": 3050890866,
5531 "max": 0,
5532 "pool": 3,
5533 "namespace": ""
5534 },
5535 "version": "65'8",
5536 "prior_version": "0'0",
5537 "last_reqid": "client.4420.0:1",
5538 "user_version": 8,
5539 "size": 7,
5540 "mtime": "2018-04-05 14:30:40.914673",
5541 "local_mtime": "2018-04-05 14:30:40.917705",
5542 "lost": 0,
5543 "flags": [
5544 "dirty",
5545 "data_digest"
5546 ],
5547 "truncate_seq": 0,
5548 "truncate_size": 0,
5549 "data_digest": "0x2ddbf8f5",
5550 "omap_digest": "0xffffffff",
5551 "expected_object_size": 0,
5552 "expected_write_size": 0,
5553 "alloc_hint_flags": 0,
5554 "manifest": {
5555 "type": 0
5556 },
5557 "watchers": {}
5558 },
5559 "shards": [
5560 {
5561 "osd": 0,
5562 "primary": false,
5563 "shard": 2,
5564 "errors": [
5565 "read_error",
5566 "hinfo_missing"
5567 ],
5568 "size": 2048
5569 },
5570 {
5571 "osd": 1,
5572 "primary": true,
5573 "shard": 0,
5574 "errors": [
5575 "read_error",
5576 "hinfo_corrupted"
5577 ],
5578 "size": 2048,
5579 "hashinfo": "bad-val"
5580 },
5581 {
5582 "osd": 2,
5583 "primary": false,
5584 "shard": 1,
5585 "errors": [],
5586 "size": 2048,
5587 "omap_digest": "0xffffffff",
5588 "data_digest": "0x04cfa72f",
5589 "hashinfo": {
5590 "cumulative_shard_hashes": [
5591 {
5592 "hash": 80717615,
5593 "shard": 0
5594 },
5595 {
5596 "hash": 1534491824,
5597 "shard": 1
5598 },
5599 {
5600 "hash": 80717615,
5601 "shard": 2
5602 }
5603 ],
5604 "total_chunk_size": 2048
5605 }
5606 }
5607 ]
5608 },
5609 {
5610 "object": {
5611 "name": "EOBJ7",
5612 "nspace": "",
5613 "locator": "",
5614 "snap": "head",
5615 "version": 10
5616 },
5617 "errors": [
5618 "hinfo_inconsistency"
5619 ],
5620 "union_shard_errors": [
5621 "ec_hash_error"
5622 ],
5623 "selected_object_info": {
5624 "oid": {
5625 "oid": "EOBJ7",
5626 "key": "",
5627 "snapid": -2,
5628 "hash": 3258066308,
5629 "max": 0,
5630 "pool": 3,
5631 "namespace": ""
5632 },
5633 "version": "75'10",
5634 "prior_version": "75'9",
5635 "last_reqid": "client.4486.0:1",
5636 "user_version": 10,
5637 "size": 34,
5638 "mtime": "2018-04-05 14:30:50.995009",
5639 "local_mtime": "2018-04-05 14:30:50.996112",
5640 "lost": 0,
5641 "flags": [
5642 "dirty",
5643 "data_digest"
5644 ],
5645 "truncate_seq": 0,
5646 "truncate_size": 0,
5647 "data_digest": "0x136e4e27",
5648 "omap_digest": "0xffffffff",
5649 "expected_object_size": 0,
5650 "expected_write_size": 0,
5651 "alloc_hint_flags": 0,
5652 "manifest": {
5653 "type": 0
5654 },
5655 "watchers": {}
5656 },
5657 "shards": [
5658 {
5659 "osd": 0,
5660 "primary": false,
5661 "shard": 2,
5662 "errors": [
5663 "ec_hash_error"
5664 ],
5665 "size": 2048,
5666 "hashinfo": {
5667 "cumulative_shard_hashes": [
5668 {
5669 "hash": 80717615,
5670 "shard": 0
5671 },
5672 {
5673 "hash": 1534491824,
5674 "shard": 1
5675 },
5676 {
5677 "hash": 80717615,
5678 "shard": 2
5679 }
5680 ],
5681 "total_chunk_size": 2048
5682 }
5683 },
5684 {
5685 "osd": 1,
5686 "primary": true,
5687 "shard": 0,
5688 "errors": [],
5689 "size": 2048,
5690 "omap_digest": "0xffffffff",
5691 "data_digest": "0x5b7455a8",
5692 "hashinfo": {
5693 "cumulative_shard_hashes": [
5694 {
5695 "hash": 1534350760,
5696 "shard": 0
5697 },
5698 {
5699 "hash": 1534491824,
5700 "shard": 1
5701 },
5702 {
5703 "hash": 1534350760,
5704 "shard": 2
5705 }
5706 ],
5707 "total_chunk_size": 2048
5708 }
5709 },
5710 {
5711 "osd": 2,
5712 "primary": false,
5713 "shard": 1,
5714 "errors": [],
5715 "size": 2048,
5716 "omap_digest": "0xffffffff",
5717 "data_digest": "0x5b7455a8",
5718 "hashinfo": {
5719 "cumulative_shard_hashes": [
5720 {
5721 "hash": 1534350760,
5722 "shard": 0
5723 },
5724 {
5725 "hash": 1534491824,
5726 "shard": 1
5727 },
5728 {
5729 "hash": 1534350760,
5730 "shard": 2
5731 }
5732 ],
5733 "total_chunk_size": 2048
5734 }
5735 }
5736 ]
5737 }
5738 ],
5739 "epoch": 0
5740 }
5741 EOF
5742
5743 fi
5744
5745 jq "$jqfilter" $dir/json | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/csjson
5746 multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
5747 if test $getjson = "yes"
5748 then
5749 if [ "$allow_overwrites" = "true" ]
5750 then
5751 num=4
5752 else
5753 num=5
5754 fi
5755 jq '.' $dir/json > save${num}.json
5756 fi
5757
5758 if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
5759 then
5760 jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
5761 fi
5762
5763 ceph osd pool rm $poolname $poolname --yes-i-really-really-mean-it
5764 }
5765
5766 function TEST_corrupt_scrub_erasure_appends() {
5767 corrupt_scrub_erasure $1 false
5768 }
5769
5770 function TEST_corrupt_scrub_erasure_overwrites() {
5771 if [ "$use_ec_overwrite" = "true" ]; then
5772 corrupt_scrub_erasure $1 true
5773 fi
5774 }
5775
5776 #
5777 # Test to make sure that a periodic scrub won't cause deep-scrub info to be lost
5778 #
5779 function TEST_periodic_scrub_replicated() {
5780 local dir=$1
5781 local poolname=psr_pool
5782 local objname=POBJ
5783
5784 run_mon $dir a --osd_pool_default_size=2 || return 1
5785 run_mgr $dir x || return 1
5786 local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 "
5787 ceph_osd_args+="--osd_scrub_backoff_ratio=0"
5788 run_osd $dir 0 $ceph_osd_args || return 1
5789 run_osd $dir 1 $ceph_osd_args || return 1
5790 create_rbd_pool || return 1
5791 wait_for_clean || return 1
5792
5793 create_pool $poolname 1 1 || return 1
5794 wait_for_clean || return 1
5795
5796 local osd=0
5797 add_something $dir $poolname $objname scrub || return 1
5798 local primary=$(get_primary $poolname $objname)
5799 local pg=$(get_pg $poolname $objname)
5800
5801 # Add deep-scrub only error
5802 local payload=UVWXYZ
5803 echo $payload > $dir/CORRUPT
5804 # Uses $ceph_osd_args for osd restart
5805 objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1
5806
5807 # No scrub information available, so expect failure
5808 set -o pipefail
5809 ! rados list-inconsistent-obj $pg | jq '.' || return 1
5810 set +o pipefail
5811
5812 pg_deep_scrub $pg || return 1
5813
5814 # Make sure bad object found
5815 rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
5816
5817 flush_pg_stats
5818 local last_scrub=$(get_last_scrub_stamp $pg)
5819 # Fake a schedule scrub
5820 ceph tell $pg scrub || return 1
5821 # Wait for schedule regular scrub
5822 wait_for_scrub $pg "$last_scrub"
5823
5824 # It needed to be upgraded
5825 grep -q "Deep scrub errors, upgrading scrub to deep-scrub" $dir/osd.${primary}.log || return 1
5826
5827 # Bad object still known
5828 rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
5829
5830 # Can't upgrade with this set
5831 ceph osd set nodeep-scrub
5832 # Let map change propagate to OSDs
5833 ceph tell osd.0 get_latest_osdmap
5834 flush_pg_stats
5835 sleep 5
5836
5837 # Fake a schedule scrub
5838 ceph tell $pg scrub || return 1
5839 # Wait for schedule regular scrub
5840 # to notice scrub and skip it
5841 local found=false
5842 for i in $(seq 14 -1 0)
5843 do
5844 sleep 1
5845 ! grep -q "Regular scrub skipped due to deep-scrub errors and nodeep-scrub set" $dir/osd.${primary}.log || { found=true ; break; }
5846 echo Time left: $i seconds
5847 done
5848 test $found = "true" || return 1
5849
5850 # Bad object still known
5851 rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1
5852
5853 flush_pg_stats
5854 # Request a regular scrub and it will be done
5855 pg_scrub $pg
5856 grep -q "Regular scrub request, deep-scrub details will be lost" $dir/osd.${primary}.log || return 1
5857
5858 # deep-scrub error is no longer present
5859 rados list-inconsistent-obj $pg | jq '.' | grep -qv $objname || return 1
5860 }
5861
5862 function TEST_scrub_warning() {
5863 local dir=$1
5864 local poolname=psr_pool
5865 local objname=POBJ
5866 local scrubs=5
5867 local deep_scrubs=5
5868 local i1_day=86400
5869 local i7_days=$(calc $i1_day \* 7)
5870 local i14_days=$(calc $i1_day \* 14)
5871 local overdue=0.5
5872 local conf_overdue_seconds=$(calc $i7_days + $i1_day + \( $i7_days \* $overdue \) )
5873 local pool_overdue_seconds=$(calc $i14_days + $i1_day + \( $i14_days \* $overdue \) )
5874
5875 run_mon $dir a --osd_pool_default_size=1 --mon_allow_pool_size_one=true || return 1
5876 run_mgr $dir x --mon_warn_pg_not_scrubbed_ratio=${overdue} --mon_warn_pg_not_deep_scrubbed_ratio=${overdue} || return 1
5877 run_osd $dir 0 $ceph_osd_args --osd_scrub_backoff_ratio=0 || return 1
5878
5879 for i in $(seq 1 $(expr $scrubs + $deep_scrubs))
5880 do
5881 create_pool $poolname-$i 1 1 || return 1
5882 wait_for_clean || return 1
5883 if [ $i = "1" ];
5884 then
5885 ceph osd pool set $poolname-$i scrub_max_interval $i14_days
5886 fi
5887 if [ $i = $(expr $scrubs + 1) ];
5888 then
5889 ceph osd pool set $poolname-$i deep_scrub_interval $i14_days
5890 fi
5891 done
5892
5893 # Only 1 osd
5894 local primary=0
5895
5896 ceph osd set noscrub || return 1
5897 ceph osd set nodeep-scrub || return 1
5898 ceph config set global osd_scrub_interval_randomize_ratio 0
5899 ceph config set global osd_deep_scrub_randomize_ratio 0
5900 ceph config set global osd_scrub_max_interval ${i7_days}
5901 ceph config set global osd_deep_scrub_interval ${i7_days}
5902
5903 # Fake schedule scrubs
5904 for i in $(seq 1 $scrubs)
5905 do
5906 if [ $i = "1" ];
5907 then
5908 overdue_seconds=$pool_overdue_seconds
5909 else
5910 overdue_seconds=$conf_overdue_seconds
5911 fi
5912 ceph tell ${i}.0 scrub $(expr ${overdue_seconds} + ${i}00) || return 1
5913 done
5914 # Fake schedule deep scrubs
5915 for i in $(seq $(expr $scrubs + 1) $(expr $scrubs + $deep_scrubs))
5916 do
5917 if [ $i = "$(expr $scrubs + 1)" ];
5918 then
5919 overdue_seconds=$pool_overdue_seconds
5920 else
5921 overdue_seconds=$conf_overdue_seconds
5922 fi
5923 ceph tell ${i}.0 deep_scrub $(expr ${overdue_seconds} + ${i}00) || return 1
5924 done
5925 flush_pg_stats
5926
5927 ceph health
5928 ceph health detail
5929 ceph health | grep -q " pgs not deep-scrubbed in time" || return 1
5930 ceph health | grep -q " pgs not scrubbed in time" || return 1
5931
5932 # note that the 'ceph tell pg deep_scrub' command now also sets the regular scrub
5933 # time-stamp. I.e. - all 'late for deep scrubbing' pgs are also late for
5934 # regular scrubbing. For now, we'll allow both responses.
5935 COUNT=$(ceph health detail | grep "not scrubbed since" | wc -l)
5936
5937 if (( $COUNT != $scrubs && $COUNT != $(expr $scrubs+$deep_scrubs) )); then
5938 ceph health detail | grep "not scrubbed since"
5939 return 1
5940 fi
5941 COUNT=$(ceph health detail | grep "not deep-scrubbed since" | wc -l)
5942 if [ "$COUNT" != $deep_scrubs ]; then
5943 ceph health detail | grep "not deep-scrubbed since"
5944 return 1
5945 fi
5946 }
5947
5948 #
5949 # Corrupt snapset in replicated pool
5950 #
5951 function TEST_corrupt_snapset_scrub_rep() {
5952 local dir=$1
5953 local poolname=csr_pool
5954 local total_objs=2
5955
5956 run_mon $dir a --osd_pool_default_size=2 || return 1
5957 run_mgr $dir x || return 1
5958 run_osd $dir 0 || return 1
5959 run_osd $dir 1 || return 1
5960 create_rbd_pool || return 1
5961 wait_for_clean || return 1
5962
5963 create_pool foo 1 || return 1
5964 create_pool $poolname 1 1 || return 1
5965 wait_for_clean || return 1
5966
5967 for i in $(seq 1 $total_objs) ; do
5968 objname=ROBJ${i}
5969 add_something $dir $poolname $objname || return 1
5970
5971 rados --pool $poolname setomapheader $objname hdr-$objname || return 1
5972 rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1
5973 done
5974
5975 local pg=$(get_pg $poolname ROBJ0)
5976 local primary=$(get_primary $poolname ROBJ0)
5977
5978 rados -p $poolname mksnap snap1
5979 echo -n head_of_snapshot_data > $dir/change
5980
5981 for i in $(seq 1 $total_objs) ; do
5982 objname=ROBJ${i}
5983
5984 # Alternate corruption between osd.0 and osd.1
5985 local osd=$(expr $i % 2)
5986
5987 case $i in
5988 1)
5989 rados --pool $poolname put $objname $dir/change
5990 objectstore_tool $dir $osd --head $objname clear-snapset corrupt || return 1
5991 ;;
5992
5993 2)
5994 rados --pool $poolname put $objname $dir/change
5995 objectstore_tool $dir $osd --head $objname clear-snapset corrupt || return 1
5996 ;;
5997
5998 esac
5999 done
6000 rm $dir/change
6001
6002 pg_scrub $pg
6003
6004 rados list-inconsistent-pg $poolname > $dir/json || return 1
6005 # Check pg count
6006 test $(jq '. | length' $dir/json) = "1" || return 1
6007 # Check pgid
6008 test $(jq -r '.[0]' $dir/json) = $pg || return 1
6009
6010 rados list-inconsistent-obj $pg > $dir/json || return 1
6011
6012 jq "$jqfilter" << EOF | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/checkcsjson
6013 {
6014 "epoch": 34,
6015 "inconsistents": [
6016 {
6017 "object": {
6018 "name": "ROBJ1",
6019 "nspace": "",
6020 "locator": "",
6021 "snap": "head",
6022 "version": 8
6023 },
6024 "errors": [
6025 "snapset_inconsistency"
6026 ],
6027 "union_shard_errors": [],
6028 "selected_object_info": {
6029 "oid": {
6030 "oid": "ROBJ1",
6031 "key": "",
6032 "snapid": -2,
6033 "hash": 1454963827,
6034 "max": 0,
6035 "pool": 3,
6036 "namespace": ""
6037 },
6038 "version": "24'8",
6039 "prior_version": "21'3",
6040 "last_reqid": "client.4195.0:1",
6041 "user_version": 8,
6042 "size": 21,
6043 "mtime": "2018-04-05 14:35:43.286117",
6044 "local_mtime": "2018-04-05 14:35:43.288990",
6045 "lost": 0,
6046 "flags": [
6047 "dirty",
6048 "omap",
6049 "data_digest"
6050 ],
6051 "truncate_seq": 0,
6052 "truncate_size": 0,
6053 "data_digest": "0x53acb008",
6054 "omap_digest": "0xffffffff",
6055 "expected_object_size": 0,
6056 "expected_write_size": 0,
6057 "alloc_hint_flags": 0,
6058 "manifest": {
6059 "type": 0
6060 },
6061 "watchers": {}
6062 },
6063 "shards": [
6064 {
6065 "osd": 0,
6066 "primary": false,
6067 "errors": [],
6068 "size": 21,
6069 "snapset": {
6070 "clones": [
6071 {
6072 "overlap": "[]",
6073 "size": 7,
6074 "snap": 1,
6075 "snaps": [
6076 1
6077 ]
6078 }
6079 ],
6080 "seq": 1
6081 }
6082 },
6083 {
6084 "osd": 1,
6085 "primary": true,
6086 "errors": [],
6087 "size": 21,
6088 "snapset": {
6089 "clones": [],
6090 "seq": 0
6091 }
6092 }
6093 ]
6094 },
6095 {
6096 "object": {
6097 "name": "ROBJ2",
6098 "nspace": "",
6099 "locator": "",
6100 "snap": "head",
6101 "version": 10
6102 },
6103 "errors": [
6104 "snapset_inconsistency"
6105 ],
6106 "union_shard_errors": [],
6107 "selected_object_info": {
6108 "oid": {
6109 "oid": "ROBJ2",
6110 "key": "",
6111 "snapid": -2,
6112 "hash": 2026323607,
6113 "max": 0,
6114 "pool": 3,
6115 "namespace": ""
6116 },
6117 "version": "28'10",
6118 "prior_version": "23'6",
6119 "last_reqid": "client.4223.0:1",
6120 "user_version": 10,
6121 "size": 21,
6122 "mtime": "2018-04-05 14:35:48.326856",
6123 "local_mtime": "2018-04-05 14:35:48.328097",
6124 "lost": 0,
6125 "flags": [
6126 "dirty",
6127 "omap",
6128 "data_digest"
6129 ],
6130 "truncate_seq": 0,
6131 "truncate_size": 0,
6132 "data_digest": "0x53acb008",
6133 "omap_digest": "0xffffffff",
6134 "expected_object_size": 0,
6135 "expected_write_size": 0,
6136 "alloc_hint_flags": 0,
6137 "manifest": {
6138 "type": 0
6139 },
6140 "watchers": {}
6141 },
6142 "shards": [
6143 {
6144 "osd": 0,
6145 "primary": false,
6146 "errors": [],
6147 "size": 21,
6148 "snapset": {
6149 "clones": [],
6150 "seq": 0
6151 }
6152 },
6153 {
6154 "osd": 1,
6155 "primary": true,
6156 "errors": [],
6157 "size": 21,
6158 "snapset": {
6159 "clones": [
6160 {
6161 "overlap": "[]",
6162 "size": 7,
6163 "snap": 1,
6164 "snaps": [
6165 1
6166 ]
6167 }
6168 ],
6169 "seq": 1
6170 }
6171 }
6172 ]
6173 }
6174 ]
6175 }
6176 EOF
6177
6178 jq "$jqfilter" $dir/json | jq '.inconsistents' | python3 -c "$sortkeys" > $dir/csjson
6179 multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1
6180 if test $getjson = "yes"
6181 then
6182 jq '.' $dir/json > save6.json
6183 fi
6184
6185 if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null;
6186 then
6187 jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
6188 fi
6189
6190 ERRORS=0
6191 declare -a err_strings
6192 err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid [0-9]*:.*:::ROBJ1:head : snapset inconsistent"
6193 err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid [0-9]*:.*:::ROBJ2:head : snapset inconsistent"
6194 err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 [0-9]*:.*:::ROBJ1:1 : is an unexpected clone"
6195 err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub : stat mismatch, got 3/4 objects, 1/2 clones, 3/4 dirty, 3/4 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 49/56 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes."
6196 err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 0 missing, 2 inconsistent objects"
6197 err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 4 errors"
6198
6199 for err_string in "${err_strings[@]}"
6200 do
6201 if ! grep -q "$err_string" $dir/osd.${primary}.log
6202 then
6203 echo "Missing log message '$err_string'"
6204 ERRORS=$(expr $ERRORS + 1)
6205 fi
6206 done
6207
6208 if [ $ERRORS != "0" ];
6209 then
6210 echo "TEST FAILED WITH $ERRORS ERRORS"
6211 return 1
6212 fi
6213
6214 ceph osd pool rm $poolname $poolname --yes-i-really-really-mean-it
6215 }
6216
6217 function TEST_request_scrub_priority() {
6218 local dir=$1
6219 local poolname=psr_pool
6220 local objname=POBJ
6221 local OBJECTS=64
6222 local PGS=8
6223
6224 run_mon $dir a --osd_pool_default_size=1 --mon_allow_pool_size_one=true || return 1
6225 run_mgr $dir x || return 1
6226 local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 "
6227 ceph_osd_args+="--osd_scrub_backoff_ratio=0"
6228 run_osd $dir 0 $ceph_osd_args || return 1
6229
6230 create_pool $poolname $PGS $PGS || return 1
6231 wait_for_clean || return 1
6232
6233 local osd=0
6234 add_something $dir $poolname $objname noscrub || return 1
6235 local primary=$(get_primary $poolname $objname)
6236 local pg=$(get_pg $poolname $objname)
6237 poolid=$(ceph osd dump | grep "^pool.*[']${poolname}[']" | awk '{ print $2 }')
6238
6239 local otherpgs
6240 for i in $(seq 0 $(expr $PGS - 1))
6241 do
6242 opg="${poolid}.${i}"
6243 if [ "$opg" = "$pg" ]; then
6244 continue
6245 fi
6246 otherpgs="${otherpgs}${opg} "
6247 local other_last_scrub=$(get_last_scrub_stamp $pg)
6248 # Fake a schedule scrub
6249 ceph tell $opg scrub $opg || return 1
6250 done
6251
6252 sleep 15
6253 flush_pg_stats
6254
6255 # Request a regular scrub and it will be done
6256 local last_scrub=$(get_last_scrub_stamp $pg)
6257 ceph pg scrub $pg
6258
6259 ceph osd unset noscrub || return 1
6260 ceph osd unset nodeep-scrub || return 1
6261
6262 wait_for_scrub $pg "$last_scrub"
6263
6264 for opg in $otherpgs $pg
6265 do
6266 wait_for_scrub $opg "$other_last_scrub"
6267 done
6268
6269 # Verify that the requested scrub ran first
6270 grep "log_channel.*scrub ok" $dir/osd.${primary}.log | grep -v purged_snaps | head -1 | sed 's/.*[[]DBG[]]//' | grep -q $pg || return 1
6271 }
6272
6273
6274 main osd-scrub-repair "$@"
6275
6276 # Local Variables:
6277 # compile-command: "cd build ; make -j4 && \
6278 # ../qa/run-standalone.sh osd-scrub-repair.sh"
6279 # End:
Ceph